{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a51097ad-531c-4bf7-813b-411b05f46300",
   "metadata": {},
   "source": [
    "# Chain of Harder Thoughts on MathQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac923168-e7f1-4c0e-ac37-f39b1b284942",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-04 19:35:55.604302: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "import re\n",
    "import json\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "33e7cf2b-db46-43c2-ae76-082a314ec10b",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a3ac5d6f-51b1-4b55-96d2-9c094a7aad0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tok2vec', <spacy.pipeline.tok2vec.Tok2Vec at 0x7f823081d0a0>),\n",
       " ('tagger', <spacy.pipeline.tagger.Tagger at 0x7f8220bf7160>),\n",
       " ('parser', <spacy.pipeline.dep_parser.DependencyParser at 0x7f81e3257cf0>),\n",
       " ('senter', <spacy.pipeline.senter.SentenceRecognizer at 0x7f823081da60>),\n",
       " ('attribute_ruler',\n",
       "  <spacy.pipeline.attributeruler.AttributeRuler at 0x7f823074adc0>),\n",
       " ('lemmatizer',\n",
       "  <spacy.lang.en.lemmatizer.EnglishLemmatizer at 0x7f8230808b00>),\n",
       " ('ner', <spacy.pipeline.ner.EntityRecognizer at 0x7f81e3257b30>)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.components"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "203a0777-c5e8-4c9b-b206-74dc21d46db9",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2d9b9d6f-eb98-4f73-8f28-b1845ef4f1c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = json.load(open('MathQA/train.json'))\n",
    "dev_data = json.load(open('MathQA/dev.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "cb0fe3bd-5cad-4399-8155-09eeebb365a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4475"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "fd108317-152f-439d-badb-3641ac563910",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev_200_idx = np.random.choice(len(dev_data), 200, replace=False)\n",
    "# np.save('MathQA/dev_200_idx', dev_200_idx)\n",
    "\n",
    "dev_200_idx = np.load('MathQA/dev_200_idx.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "fffe4847-76b5-4a57-9f41-77f46b445fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data_200 = []\n",
    "for i in dev_200_idx: dev_data_200.append(dev_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b6c56787-f397-431e-821b-355b64413cb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"3 pieces of 1 / 5 length and two piece each of 1 / 15 and 2 / 15 lengths .\n",
      "removing one piece each from pieces of each kind of lengths the all that will remain will be 2 pieces of 1 / 5 i .\n",
      "e 2 / 5 , 1 piece of 1 / 15 , and 1 piece of 2 / 15 which gives us 2 / 5 + 1 / 15 + 2 / 15 - - - - - > 3 / 5 answer is d\"\n"
     ]
    }
   ],
   "source": [
    "idx = 20\n",
    "doc = nlp(train_data[idx]['Rationale'])\n",
    "for s in doc.sents: print(s.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8f480999-b1d9-4744-9407-faacd66823f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Problem': 'kim finds a 5 - meter tree branch and marks it off in thirds and fifths . she then breaks the branch along all the markings and removes one piece of every distinct length . what fraction of the original branch remains ?',\n",
       " 'Rationale': '\"3 pieces of 1 / 5 length and two piece each of 1 / 15 and 2 / 15 lengths . removing one piece each from pieces of each kind of lengths the all that will remain will be 2 pieces of 1 / 5 i . e 2 / 5 , 1 piece of 1 / 15 , and 1 piece of 2 / 15 which gives us 2 / 5 + 1 / 15 + 2 / 15 - - - - - > 3 / 5 answer is d\"',\n",
       " 'options': 'a ) 8 / 15 , b ) 1 / 2 , c ) 7 / 5 , d ) 3 / 5 , e ) 7 / 5',\n",
       " 'correct': 'd',\n",
       " 'annotated_formula': 'subtract(const_1, add(add(divide(5, multiply(add(const_2, 5), 5)), divide(const_2, multiply(add(const_2, 5), 5))), divide(const_1, multiply(add(const_2, 5), 5))))',\n",
       " 'linear_formula': 'add(const_2,n0)|multiply(n0,#0)|divide(n0,#1)|divide(const_2,#1)|divide(const_1,#1)|add(#2,#3)|add(#5,#4)|subtract(const_1,#6)|',\n",
       " 'category': 'physics'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f6b1d003-6b0f-4b6b-b593-697f787a53c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"usually we can solve every question of this type by choosing appropriate value of the variable and deriving the value of other related variables . let , p = 400 then j = ( 75 / 100 ) * 400 = 300 also j = ( 80 / 100 ) * t i . e . t = 300 * 100 / 80 = 375 and t = [ 1 - ( q / 100 ) ] * p i . e . 100 - q = 100 * t / p = 100 * 375 / 400 = 93.75 i . e . q = 6.25 answer : option d\"'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[8]['Rationale']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2989b102-ec3f-477a-a4ad-ab5c796aff39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"usually we can solve every question of this type by choosing appropriate value of the variable and deriving the value of other related variables ',\n",
       " ' let , p = 400 then j = ( 75 / 100 ) * 400 = 300 also j = ( 80 / 100 ) * t i ',\n",
       " ' e ',\n",
       " ' t = 300 * 100 / 80 = 375 and t = [ 1 - ( q / 100 ) ] * p i ',\n",
       " ' e ',\n",
       " ' 100 - q = 100 * t / p = 100 * 375 / 400 = 93',\n",
       " '75 i ',\n",
       " ' e ',\n",
       " ' q = 6',\n",
       " '25 answer : option d\"']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[8]['Rationale'].split('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "970064d2-21c8-4fc4-a30f-e5372e82fd9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Problem': 'an art gallery has only paintings and sculptures . currently , 1 / 3 of the pieces of art are displayed , and 1 / 6 of the pieces on display are sculptures . if 1 / 3 of the pieces not on display are paintings , and 800 sculptures are not on display , how many pieces of art does the gallery have ?',\n",
       " 'Rationale': '\"too many words and redundant info there . ( i ) 1 / 3 of the pieces of art are displayed , hence 2 / 3 of the pieces of art are not displayed . ( ii ) 1 / 6 of the pieces on display are sculptures , hence 5 / 6 of the pieces on display are paintings . ( iii ) 1 / 3 of the pieces not on display are paintings , hence 2 / 3 of the pieces not on display are sculptures . 800 sculptures are not on display , so according to ( iii ) 2 / 3 * { not on display } = 800 - - > { not on display } = 1200 . according to ( i ) 2 / 3 * { total } = 1200 - - > { total } = 1800 . answer : b .\"',\n",
       " 'options': 'a ) 360 , b ) 1800 , c ) 540 , d ) 640 , e ) 720',\n",
       " 'correct': 'b',\n",
       " 'annotated_formula': 'divide(divide(800, subtract(1, divide(1, 3))), subtract(1, divide(1, 3)))',\n",
       " 'linear_formula': 'divide(n0,n1)|subtract(n0,#0)|divide(n6,#1)|divide(#2,#1)|',\n",
       " 'category': 'general'}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_data[8]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1a3884c-5d16-4386-b7af-f686e6eeef8c",
   "metadata": {},
   "source": [
    "## Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "562c6f3d-302b-49a0-9be9-10bd2b7e6394",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_step(sent):\n",
    "    doc = nlp(sent)\n",
    "    n_step = len(list(s for s in doc.sents))\n",
    "    return n_step\n",
    "\n",
    "def split_step(sent):\n",
    "    doc = nlp(sent)\n",
    "    steps = list(s for s in doc.sents)\n",
    "    return steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "id": "180d846c-9637-4e29-a29d-c91c383d040b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ans_dict(c):\n",
    "    if(c.startswith(\"['\")):\n",
    "        c_ = eval(c)\n",
    "        ans_dict = {}\n",
    "        for ac in c_:\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "    else:\n",
    "        ans_dict = {}\n",
    "        pattern = ' , [a-e] \\) '\n",
    "        c_ = re.split(pattern, c)\n",
    "        c_[0] = c_[0].split(' ) ')[-1]\n",
    "        for i, c in zip(['a', 'b', 'c', 'd', 'e'], c_):\n",
    "            ans_dict[i] = c\n",
    "    return ans_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c82f75bf-b3eb-4a1a-9d0b-1673e30c8d34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_step(train_data[8]['Rationale'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f1dce8fc-af77-4c67-94d4-22033b332601",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29837/29837 [04:56<00:00, 100.54it/s]\n"
     ]
    }
   ],
   "source": [
    "train_steps = []\n",
    "for d in tqdm(train_data): train_steps.append(get_step(d['Rationale']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "fce22f07-6f5a-4e73-a496-1f634f5d82f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29837/29837 [00:00<00:00, 370512.60it/s]\n"
     ]
    }
   ],
   "source": [
    "train_ans_lens = []\n",
    "for d in tqdm(train_data): train_ans_lens.append(len(d['Rationale']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ef9c52e4-0209-489b-afad-9b41f1b7173e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  1.,  2.,  2.,  3.,  4.,  5.,  6.,  8., 18., 85.])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(train_steps, [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 99, 100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c520768f-8697-4adb-986e-1626d943ebae",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step_dict = {}\n",
    "for i, ti in enumerate(train_steps):\n",
    "    if ti in train_step_dict: train_step_dict[ti].append(i)\n",
    "    else: train_step_dict[ti] = [i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "df7ac19d-6fe7-4802-944a-b4988d886137",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[240,\n",
       " 264,\n",
       " 317,\n",
       " 1588,\n",
       " 2191,\n",
       " 2490,\n",
       " 3123,\n",
       " 3395,\n",
       " 3689,\n",
       " 4265,\n",
       " 4806,\n",
       " 4823,\n",
       " 5056,\n",
       " 5284,\n",
       " 5394,\n",
       " 6140,\n",
       " 6312,\n",
       " 6571,\n",
       " 7236,\n",
       " 7583,\n",
       " 7629,\n",
       " 8366,\n",
       " 8655,\n",
       " 10494,\n",
       " 12557,\n",
       " 12834,\n",
       " 13186,\n",
       " 14116,\n",
       " 14377,\n",
       " 14951,\n",
       " 15066,\n",
       " 15559,\n",
       " 16055,\n",
       " 16833,\n",
       " 17398,\n",
       " 17443,\n",
       " 18129,\n",
       " 18679,\n",
       " 19140,\n",
       " 20276,\n",
       " 21745,\n",
       " 22191,\n",
       " 22763,\n",
       " 22767,\n",
       " 22905,\n",
       " 23974,\n",
       " 24863,\n",
       " 25242,\n",
       " 25627,\n",
       " 26013,\n",
       " 26481,\n",
       " 26658,\n",
       " 27436,\n",
       " 27515,\n",
       " 27902,\n",
       " 28797,\n",
       " 28921]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_step_dict[18]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5ebff2b1-e5f2-42bf-b2ae-debdb792cc88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"you can simply use this formula to avoid confusion : w 1 / w 2 = ( a 2 - aavg ) / ( avg - a 1 ) here is how you will find the values of a 1 an a 2 . we have an overall loss ( average loss ) . the average loss is 5 kg when 50 kg alloy is immersed . this is a loss of ( 5 / 50 ) * 100 = 10 % . this is aavg the loss of tin is 1.375 kg for every 10 kg . this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water . this is a 1 . the loss of silver is . 375 kg for every 5 kg . this means it loses ( . 375 / 5 ) * 100 = 7.5 % of its weight in water . this is a 2 . weight of tin / weight of silver = ( silver \\' s loss - avg loss ) / ( avg loss - tin \\' s loss ) x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3 e\"'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_step_dict[18][0]]['Rationale']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "87776465-20df-430d-9f8f-eebb9ab14a4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"you can simply use this formula to avoid confusion : w 1 / w 2 = ( a 2 - aavg ) / ( avg - a 1 ) here is how you will find the values of a 1 an a 2 .,\n",
       " we have an overall loss ( average loss ) .,\n",
       " the average loss is 5 kg when 50 kg alloy is immersed .,\n",
       " this is a loss of ( 5 / 50 ) *,\n",
       " 100 = 10 % .,\n",
       " this is aavg the loss of tin is 1.375 kg for every 10 kg .,\n",
       " this means it loses ( 1.375 / 10 ) *,\n",
       " 100 = 13.75 % of its weight in water .,\n",
       " this is a 1 .,\n",
       " the loss of silver is .,\n",
       " 375 kg for every 5 kg .,\n",
       " this means it loses ( .,\n",
       " 375 / 5 ) *,\n",
       " 100 = 7.5 % of its weight in water .,\n",
       " this is a 2 . weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss ),\n",
       " x / y,\n",
       " = ( 7.5 - 10 ) / ( 10 - 13.75 ),\n",
       " = 2 / 3 e\"]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_step(train_data[train_step_dict[18][0]]['Rationale'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b5e430c-34c0-46d1-b371-5a48d20ca511",
   "metadata": {},
   "source": [
    "# Chain of Harder Thoughts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "10ec14f4-9fca-4044-8ae7-5a1baf893885",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_answer(pred_str, ans_str):\n",
    "    \"\"\"Find the last number as the predicted answer\"\"\"\n",
    "    pattern = '\\d*\\.?\\d+'\n",
    "    pred = re.findall(pattern, pred_str)\n",
    "    if(len(pred) >= 1):\n",
    "        # print(pred_str)\n",
    "        pred = pred[-1]\n",
    "        gold = re.findall(pattern, ans_str)\n",
    "        # print(ans_str)\n",
    "        gold = gold[-1]\n",
    "        return pred == gold\n",
    "    else: return False\n",
    "\n",
    "def parse_pred_ans(filename):\n",
    "    with open(filename) as fd: lines = fd.readlines()\n",
    "    am, a = None, None\n",
    "    num_q, acc = 0, 0\n",
    "    current_mode = 'none'\n",
    "    questions = []\n",
    "    ans_pred = []\n",
    "    ans_gold = []\n",
    "    for l in lines:\n",
    "        if(l.startswith('Question: ')):\n",
    "            if(am is not None and a is not None):\n",
    "                questions.append(q)\n",
    "                ans_pred.append(am)\n",
    "                ans_gold.append(a)\n",
    "                if(test_answer(am, a)):\n",
    "                    acc += 1\n",
    "            current_mode = 'q'\n",
    "            q = l\n",
    "            num_q += 1\n",
    "        elif(l.startswith('Options: ')): pass\n",
    "        elif(l.startswith('A_model:')):\n",
    "            current_mode = 'am'\n",
    "            am = l\n",
    "        elif(l.startswith('A:')):\n",
    "            current_mode = 'a'\n",
    "            a = l\n",
    "        else:\n",
    "            if(current_mode == 'q'): q += l\n",
    "            elif(current_mode == 'am'): am += l\n",
    "            elif(current_mode == 'a'): a += l\n",
    "            else:\n",
    "                raise ValueError(current_mode)\n",
    "                \n",
    "    questions.append(q)\n",
    "    ans_pred.append(am)\n",
    "    ans_gold.append(a)\n",
    "    if(test_answer(am, a)):\n",
    "        acc += 1\n",
    "    print('num_q %d correct %d ratio %.4f' % (num_q, acc, float(acc / num_q)))\n",
    "    return questions, ans_pred, ans_gold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d8561d5-c429-4086-8246-e2c44d314fdd",
   "metadata": {},
   "source": [
    "## Easier, sentence approx 1 and short, avg step 1.625, acc 38"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "dac9dc98-710e-4a8e-8311-fdf85d63578c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# easiest_idx = np.random.choice(len(train_step_dict[1]), 3 * 8, replace=False)\n",
    "easiest_idx = np.argsort(train_ans_lens)[2500:2512]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "62f594b1-a987-4c30-ac5a-d77a46e0f878",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11809, 10101, 14646, 15956,  4681,  8446, 12433, 26106, 26671,\n",
       "       16851,  3142, 15785])"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "easiest_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "5d01ecdb-cc1f-49ac-90f5-c2a3883b4823",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easiest_1 = ''\n",
    "\n",
    "for ei in easiest_idx:\n",
    "    idx = ei\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_easiest_1 += 'Question: ' + q + '\\n'\n",
    "    prompt_easiest_1 += 'Options: ' + c + '\\n'\n",
    "    prompt_easiest_1 += \"Let's think step by step\\n\"\n",
    "    prompt_easiest_1 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "21bcb08d-256a-45b8-838a-97e62ab571f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a certain number when divided by 39 leaves a remainder 19 , what is the remainder when the same number is divided by 13 ?\n",
      "Options: a ) 7 , b ) 8 , c ) 9 , d ) 6 , e ) 4\n",
      "Let's think step by step\n",
      "explanation : 39 + 19 = 58 / 13 = 6 ( remainder ) answer : d 6 \n",
      "\n",
      "Question: 10 % people of a village in sri lanka died by bombardment , 15 % of the remainder left the village on account of fear . if now the population is reduced to 3213 , how much was it in the beginning ?\n",
      "Options: a ) a ) 3800 , b ) b ) 4200 , c ) c ) 4400 , d ) d ) 4500 , e ) e ) 4600\n",
      "Let's think step by step\n",
      "x * ( 90 / 100 ) * ( 85 / 100 ) = 3213 x = 4200 answer : b 4200 \n",
      "\n",
      "Question: tim came second in math . when his mother asked him how much he had scored , he answered that he got the sum of the first 11 even numbers . his mother immediately worked out the answer . how much had he scored in math ?\n",
      "Options: a ) 80 , b ) 190 , c ) 132 , d ) 170 , e ) 60\n",
      "Let's think step by step\n",
      "c 132 sum = ( n x n ) + n hence , 11 x 11 = 121 + 11 = 132 132 \n",
      "\n",
      "Question: in a class of 38 children , 19 play tennis , 21 play squash and 10 play neither sport . ( 1 ) how many children play both sports ?\n",
      "Options: a ) 11 , b ) 12 , c ) 14 , d ) 15 , e ) 19\n",
      "Let's think step by step\n",
      "28 play some sport 19 + 21 - both = 28 both = 12 answer is b 12 \n",
      "\n",
      "Question: jim is able to sell a hand - carved statue for $ 540 which was a 35 % profit over his cost . how much did the statue originally cost him ?\n",
      "Options: a ) $ 400.00 , b ) $ 412.40 , c ) $ 455.40 , d ) $ 474.90 , e ) $ 488.20\n",
      "Let's think step by step\n",
      "540 = 1.35 * x x = 540 / 1.35 = 400 $ 400 , which is ( a ) . $ 400.00 \n",
      "\n",
      "Question: find the mean proportional between 49 & 81 ?\n",
      "Options: a ) 59 , b ) 61 , c ) 63 , d ) 65 , e ) 67\n",
      "Let's think step by step\n",
      "formula = √ a × b a = 49 and b = 81 √ 49 × 81 = 7 × 9 = 63 c 63 \n",
      "\n",
      "Question: a began business with rs . 27000 and was joined afterwards by b with rs . 36000 . when did b join if the profits at the end of the year were divided in the ratio of 2 : 1 ?\n",
      "Options: a ) 1.5 , b ) 6.5 , c ) 7.5 , d ) 8 , e ) 2\n",
      "Let's think step by step\n",
      "27 * 12 : 36 * x = 2 : 1 x = 4.5 12 - 4.5 = 7.5 answer : c 7.5 \n",
      "\n",
      "Question: two trains 111 meters and 165 meters in length respectively are running in opposite directions , one at the rate of 80 km and the other at the rate of 65 kmph . in what time will they be completely clear of each other from the moment they meet ?\n",
      "Options: a ) 4.85 , b ) 7.85 , c ) 6.85 , d ) 5.85 , e ) 6.15\n",
      "Let's think step by step\n",
      "t = ( 111 + 165 ) / ( 80 + 65 ) * 18 / 5 t = 6.85 answer : c 6.85 \n",
      "\n",
      "Question: in a games hour 4 different types of players came to the ground ? cricket 10 , hokey 12 , football 16 , softball 13 . how many players are present in the ground ?\n",
      "Options: a ) 70 , b ) 52 , c ) 51 , d ) 49 , e ) 50\n",
      "Let's think step by step\n",
      "total number of players = 10 + 12 + 16 + 13 = 51 answer is c 51 \n",
      "\n",
      "Question: 10 % people of a village in sri lanka died by bombardment , 20 % of the remainder left the village on account of fear . if now the population is reduced to 3240 , how much was it in the beginning ?\n",
      "Options: a ) a ) 3800 , b ) b ) 4200 , c ) c ) 4400 , d ) d ) 4500 , e ) e ) 4600\n",
      "Let's think step by step\n",
      "x * ( 90 / 100 ) * ( 80 / 100 ) = 3240 x = 4500 answer : d 4500 \n",
      "\n",
      "Question: find the length of the wire required to go 10 times round a square field containing 53824 m 2 .\n",
      "Options: ['a ) 15840', 'b ) 9280', 'c ) 2667', 'd ) 8766', 'e ) 66711']\n",
      "Let's think step by step\n",
      "a 2 = 53824 = > a = 232 4 a = 928 928 * 10 = 9280 answer : b 9280'\n",
      "\n",
      "Question: if 16 ^ y = 2 ^ 16 , what is y ?\n",
      "Options: a ) 2 , b ) 4 , c ) 8 , d ) 10 , e ) 12\n",
      "Let's think step by step\n",
      "16 ^ y = 2 ^ 4 y = 2 ^ 16 4 y = 16 y = 4 the answer is b . 4 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_easiest_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "9404cc0c-7027-411f-bbdc-4418341022d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easiest_1_lb = [\n",
    "\"Question: a certain number when divided by 39 leaves a remainder 19 , what is the remainder when the same number is divided by 13 ?\\n\",\n",
    "\"Options: a ) 7 , b ) 8 , c ) 9 , d ) 6 , e ) 4\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"39 + 19 = 58 / 13 = 6 ( remainder )\\n\", \n",
    "\"The answer is d, 6\\n\\n\",\n",
    "\n",
    "\"Question: 10 % people of a village in sri lanka died by bombardment , 15 % of the remainder left the village on account of fear . if now the population is reduced to 3213 , how much was it in the beginning ?\\n\",\n",
    "\"Options: a ) a ) 3800 , b ) b ) 4200 , c ) c ) 4400 , d ) d ) 4500 , e ) e ) 4600\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"x * ( 90 / 100 ) * ( 85 / 100 ) = 3213\\n\",\n",
    "\"x = 4200\\n\",\n",
    "\"The answer is, b 4200\\n\\n\"\n",
    "\n",
    "\"Question: tim came second in math . when his mother asked him how much he had scored , he answered that he got the sum of the first 11 even numbers . his mother immediately worked out the answer . how much had he scored in math ?\\n\"\n",
    "\"Options: a ) 80 , b ) 190 , c ) 132 , d ) 170 , e ) 60\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"sum = ( n x n ) + n\\n\",\n",
    "\"hence , 11 x 11 = 121 + 11 = 132\\n\",\n",
    "\"The answer is c, 132\\n\\n\",\n",
    "\n",
    "\"Question: in a class of 38 children , 19 play tennis , 21 play squash and 10 play neither sport . ( 1 ) how many children play both sports ?\\n\",\n",
    "\"Options: a ) 11 , b ) 12 , c ) 14 , d ) 15 , e ) 19\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"28 play some sport 19 + 21 - both = 28\\n\",\n",
    "\"both = 12\\n\",\n",
    "\"The answer is b, 12\\n\\n\",\n",
    "\n",
    "\"Question: jim is able to sell a hand - carved statue for $ 540 which was a 35 % profit over his cost . how much did the statue originally cost him ?\\n\",\n",
    "\"Options: a ) $ 400.00 , b ) $ 412.40 , c ) $ 455.40 , d ) $ 474.90 , e ) $ 488.20\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"540 = 1.35 * x\\n\",\n",
    "\"x = 540 / 1.35 = 400\\n\",\n",
    "\"The answer is a, $ 400.00\\n\\n\",\n",
    "\n",
    "\"Question: find the mean proportional between 49 & 81 ?\\n\"\n",
    "\"Options: a ) 59 , b ) 61 , c ) 63 , d ) 65 , e ) 67\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"formula = √ a × b, a = 49 and b = 81m, √ 49 × 81 = 7 × 9 = 63\\n\",\n",
    "\"The answer is c, 63\\n\\n\"\n",
    "\n",
    "\"Question: a began business with rs . 27000 and was joined afterwards by b with rs . 36000 . when did b join if the profits at the end of the year were divided in the ratio of 2 : 1 ?\\n\"\n",
    "\"Options: a ) 1.5 , b ) 6.5 , c ) 7.5 , d ) 8 , e ) 2\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"27 * 12 : 36 * x = 2 : 1\\n\",\n",
    "\"x = 4.5, 12 - 4.5 = 7.5\\n\",\n",
    "\"The answer is c, 7.5\\n\\n\"\n",
    "\n",
    "\"Question: two trains 111 meters and 165 meters in length respectively are running in opposite directions , one at the rate of 80 km and the other at the rate of 65 kmph . in what time will they be completely clear of each other from the moment they meet ?\\n\"\n",
    "\"Options: a ) 4.85 , b ) 7.85 , c ) 6.85 , d ) 5.85 , e ) 6.15\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"t = ( 111 + 165 ) / ( 80 + 65 ) * 18 / 5, t = 6.85\\n\",\n",
    "\"The answer is c, 6.85\\n\\n\"\n",
    "\n",
    "# Question: in a games hour 4 different types of players came to the ground ? cricket 10 , hokey 12 , football 16 , softball 13 . how many players are present in the ground ?\n",
    "# Options: a ) 70 , b ) 52 , c ) 51 , d ) 49 , e ) 50\n",
    "# Let's think step by step\n",
    "# total number of players = 10 + 12 + 16 + 13 = 51 answer is c 51 \n",
    "\n",
    "# Question: 10 % people of a village in sri lanka died by bombardment , 20 % of the remainder left the village on account of fear . if now the population is reduced to 3240 , how much was it in the beginning ?\n",
    "# Options: a ) a ) 3800 , b ) b ) 4200 , c ) c ) 4400 , d ) d ) 4500 , e ) e ) 4600\n",
    "# Let's think step by step\n",
    "# x * ( 90 / 100 ) * ( 80 / 100 ) = 3240 x = 4500 answer : d 4500 \n",
    "\n",
    "# Question: find the length of the wire required to go 10 times round a square field containing 53824 m 2 .\n",
    "# Options: ['a ) 15840', 'b ) 9280', 'c ) 2667', 'd ) 8766', 'e ) 66711']\n",
    "# Let's think step by step\n",
    "# a 2 = 53824 = > a = 232 4 a = 928 928 * 10 = 9280 answer : b 9280'\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "8cedaecc-96e7-42e9-8aa4-119b593de41b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.625\n"
     ]
    }
   ],
   "source": [
    "step_len = (1 + 2 + 2 + 2 + 2 + 1 + 2 + 1) / 8\n",
    "print(step_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "3e6426aa-283f-4bc0-8cfe-e0d1e492422d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a certain number when divided by 39 leaves a remainder 19 , what is the remainder when the same number is divided by 13 ?\n",
      "Options: a ) 7 , b ) 8 , c ) 9 , d ) 6 , e ) 4\n",
      "Let's think step by step\n",
      "39 + 19 = 58 / 13 = 6 ( remainder )\n",
      "The answer is d, 6\n",
      "\n",
      "Question: 10 % people of a village in sri lanka died by bombardment , 15 % of the remainder left the village on account of fear . if now the population is reduced to 3213 , how much was it in the beginning ?\n",
      "Options: a ) a ) 3800 , b ) b ) 4200 , c ) c ) 4400 , d ) d ) 4500 , e ) e ) 4600\n",
      "Let's think step by step\n",
      "x * ( 90 / 100 ) * ( 85 / 100 ) = 3213\n",
      "x = 4200\n",
      "The answer is, b 4200\n",
      "\n",
      "Question: tim came second in math . when his mother asked him how much he had scored , he answered that he got the sum of the first 11 even numbers . his mother immediately worked out the answer . how much had he scored in math ?\n",
      "Options: a ) 80 , b ) 190 , c ) 132 , d ) 170 , e ) 60\n",
      "Let's think step by step\n",
      "sum = ( n x n ) + n\n",
      "hence , 11 x 11 = 121 + 11 = 132\n",
      "The answer is c, 132\n",
      "\n",
      "Question: in a class of 38 children , 19 play tennis , 21 play squash and 10 play neither sport . ( 1 ) how many children play both sports ?\n",
      "Options: a ) 11 , b ) 12 , c ) 14 , d ) 15 , e ) 19\n",
      "Let's think step by step\n",
      "28 play some sport 19 + 21 - both = 28\n",
      "both = 12\n",
      "The answer is b, 12\n",
      "\n",
      "Question: jim is able to sell a hand - carved statue for $ 540 which was a 35 % profit over his cost . how much did the statue originally cost him ?\n",
      "Options: a ) $ 400.00 , b ) $ 412.40 , c ) $ 455.40 , d ) $ 474.90 , e ) $ 488.20\n",
      "Let's think step by step\n",
      "540 = 1.35 * x\n",
      "x = 540 / 1.35 = 400\n",
      "The answer is a, $ 400.00\n",
      "\n",
      "Question: find the mean proportional between 49 & 81 ?\n",
      "Options: a ) 59 , b ) 61 , c ) 63 , d ) 65 , e ) 67\n",
      "Let's think step by step\n",
      "formula = √ a × b, a = 49 and b = 81m, √ 49 × 81 = 7 × 9 = 63\n",
      "The answer is c, 63\n",
      "\n",
      "Question: a began business with rs . 27000 and was joined afterwards by b with rs . 36000 . when did b join if the profits at the end of the year were divided in the ratio of 2 : 1 ?\n",
      "Options: a ) 1.5 , b ) 6.5 , c ) 7.5 , d ) 8 , e ) 2\n",
      "Let's think step by step\n",
      "27 * 12 : 36 * x = 2 : 1\n",
      "x = 4.5, 12 - 4.5 = 7.5\n",
      "The answer is c, 7.5\n",
      "\n",
      "Question: two trains 111 meters and 165 meters in length respectively are running in opposite directions , one at the rate of 80 km and the other at the rate of 65 kmph . in what time will they be completely clear of each other from the moment they meet ?\n",
      "Options: a ) 4.85 , b ) 7.85 , c ) 6.85 , d ) 5.85 , e ) 6.15\n",
      "Let's think step by step\n",
      "t = ( 111 + 165 ) / ( 80 + 65 ) * 18 / 5, t = 6.85\n",
      "The answer is c, 6.85\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join(prompt_easiest_1_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "ea806ac2-250a-40ed-98d6-5adffa86fb3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [09:01<00:00,  2.71s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_easiest_1_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_easiest_1_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "ce4f3f0e-6dba-4f90-81a3-dbb8f5011fe2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 76 ratio 0.3800\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_easiest_1, ans_easiest_hard_1 =parse_pred_ans('outputs/dev_outputs_coht_easiest_1_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ea86497-7e55-4945-bcb6-8de7666661b9",
   "metadata": {},
   "source": [
    "## Easier, sentence approx 1, slightly longer, avg step 2.25, acc 34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "cdad61b6-9828-4f17-8a6d-b61dcc4a87e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# easier_idx = np.random.choice(len(train_step_dict[1]), 3 * 8, replace=False)\n",
    "easier_idx = np.array([220, 5061, 6473, 3452, 7221,  533, 4781, 4128, 1092, 1947, 3995,\n",
    "       3084, 2567, 3596, 3663, 2516, 1434, 5472, 2587, 1946, 3291, 4393,\n",
    "       3401, 2629])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "f9d639da-ca02-48c0-9823-a1575e11fae0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 220, 5061, 6473, 3452, 7221,  533, 4781, 4128, 1092, 1947, 3995,\n",
       "       3084, 2567, 3596, 3663, 2516, 1434, 5472, 2587, 1946, 3291, 4393,\n",
       "       3401, 2629])"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "easier_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "f3c659be-ae0f-4f64-a0cc-d2f188a67d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "easier_idx_1 = easier_idx[:8]\n",
    "easier_idx_2 = easier_idx[8: 16]\n",
    "easier_idx_3 = easier_idx[16:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "3cfe5390-e4a9-45f3-9ef7-a237f3b87eb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easier_1 = ''\n",
    "\n",
    "for ei in easier_idx_1:\n",
    "    idx = train_step_dict[1][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_easier_1 += 'Question: ' + q + '\\n'\n",
    "    prompt_easier_1 += 'Options: ' + c + '\\n'\n",
    "    prompt_easier_1 += \"Let's think step by step\\n\"\n",
    "    prompt_easier_1 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "eec04b28-77d1-41b6-abb6-6a4d45946f0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: if a = { 10 , 15 , 18 , 22 , 36 , 44 } , what is the product of mean and median of the numbers in a ?\n",
      "Options: a ) 105 , b ) 245 , c ) 360 , d ) 480 , e ) 542\n",
      "Let's think step by step\n",
      "mean = ( 10 + 15 + 18 + 22 + 36 + 43 ) / 6 = 24 median = ( 18 + 22 ) / 2 = 20 product = 24 * 20 = 480 option d 480 \n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "58 / 4 = 14.5 gallons 14.5 * 32 = 464 miles the answer is c . 464 \n",
      "\n",
      "Question: how many of the positive factors of 36 , 84 and how many common factors are there in numbers ?\n",
      "Options: a ) 2 , b ) 4 , c ) 3 , d ) 5 , e ) 6\n",
      "Let's think step by step\n",
      "factors of 36 - 1 , 2 , 3 , 4 , 6 , 9 , 12 , 18 , 36 factors of 84 - 1 , 2 , 3 , 4 , 6 , 7 , 12 , 14 , 21 , 28 , 42 , 84 comparing both , we have six common factors of 36 and 84 - 1,2 , 3,4 , 6,12 , answer ( e ) 6\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "r = 7 h = 5 2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528 answer : b 528 \n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "probability of first member an english teacher = 3 / 9 probability of second member an english teacher = 2 / 8 probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12 ( e ) 1 / 12\n",
      "\n",
      "Question: a , b , c , d and e are ve consecutive odd numbers the sum of a and c is 146 . what is the value of e ?\n",
      "Options: a ) 75 , b ) 81 , c ) 71 , d ) 79 , e ) none of these\n",
      "Let's think step by step\n",
      "a + c = 146 ⇒ a + a + 4 = 146 ⇒ a = 146 - 4 / 2 = 71 ∴ e = a + 8 = 71 + 8 = 79 answer d 79 \n",
      "\n",
      "Question: ( 8956231 x 465 ) = ?\n",
      "Options: a ) 4584638125 , b ) 4164647415 , c ) 4584638135 , d ) 4584638140 , e ) 4584638145\n",
      "Let's think step by step\n",
      "8956231 x 465 = 4164647415 ans b 4164647415 \n",
      "\n",
      "Question: reduce 368 / 598 to the lowest terms\n",
      "Options: a ) 30 / 25 , b ) 28 / 13 , c ) 28 / 29 , d ) 8 / 13 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : we can do it easily by in two steps step 1 : we get the hcf of 368 and 598 which is 46 step 2 : divide both by 46 , we will get the answer 8 / 13 answer : option d 8 / 13 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_easier_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "fbc266a3-06ce-4b4d-94b5-a54d3499b6f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easier_1_lb = [\n",
    "\"Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\\n\",\n",
    "\"Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"12 * 10 + 30 = 150.\\n\",\n",
    "\"( 30 / 130 ) * 100 = 23.1\\n\",\n",
    "\"The answer is c, 23.1 %\\n\\n\",\n",
    "\n",
    "\"Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\\n\",\n",
    "\"Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600\\n\",\n",
    "\"26 x^2 = 2600, x^2 = 100, x = 10\\n\",\n",
    "\"The answer is a, 10\\n\\n\",\n",
    "\n",
    "\"Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\\n\"\n",
    "\"Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"total weight increased = ( 8 x 1.5 ) kg = 6 kg\\n\",\n",
    "\"weight of new person = ( 75 + 6 ) kg = 81 kg.\\n\",\n",
    "\"The answer is a, 81 kg\\n\\n\"\n",
    "\n",
    "\"Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\\n\",\n",
    "\"Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"1 also gives the remainder of 1 when divided by 3 .\\n\",\n",
    "\"so , there are total of 20 numbers .\\n\",\n",
    "\"The answer is e, 20\\n\\n\"\n",
    "\n",
    "\"Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\\n\"\n",
    "\"Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"2a / 100 = 8, a = 400\\n\",\n",
    "\"8b / 100 = 2, b = 25\\n\",\n",
    "\"c = b / a = 25 / 400 = 1 / 16\\n\",\n",
    "\"The answer is a, 1 / 16\\n\\n\"\n",
    "\n",
    "\"Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\\n\"\n",
    "\"Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"probability of first member an english teacher = 3 / 9\\n\",\n",
    "\"probability of second member an english teacher = 2 / 8\\n\",\n",
    "\"probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\\n\",\n",
    "\"The answer is e, 1 / 12\\n\\n\"\n",
    "    \n",
    "\"Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\\n\"\n",
    "\"Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"r = 7, h = 5\\n\",\n",
    "\"2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\\n\",\n",
    "\"The answer is b, 528\\n\\n\"\n",
    "    \n",
    "\"Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\\n\"\n",
    "\"Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"58 / 4 = 14.5 gallons\\n\",\n",
    "\"14.5 * 32 = 464 miles\\n\",\n",
    "\"the answer is c, 464\\n\\n\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "0294f003-cd4f-4402-8962-555ca487f0ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.25\n"
     ]
    }
   ],
   "source": [
    "step_len = (2 + 2 + 2 + 2 + 3 + 3 + 2 + 2) / 8 \n",
    "print(step_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "cd33b5c3-9866-43ef-bed4-ac0da8317c37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\n",
      "Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\n",
      "Let's think step by step\n",
      "12 * 10 + 30 = 150.\n",
      "( 30 / 130 ) * 100 = 23.1\n",
      "The answer is c, 23.1 %\n",
      "\n",
      "Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\n",
      "Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\n",
      "Let's think step by step\n",
      "let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600\n",
      "26 x^2 = 2600, x^2 = 100, x = 10\n",
      "The answer is a, 10\n",
      "\n",
      "Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\n",
      "Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\n",
      "Let's think step by step\n",
      "total weight increased = ( 8 x 1.5 ) kg = 6 kg\n",
      "weight of new person = ( 75 + 6 ) kg = 81 kg.\n",
      "The answer is a, 81 kg\n",
      "\n",
      "Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\n",
      "Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\n",
      "Let's think step by step\n",
      "1 also gives the remainder of 1 when divided by 3 .\n",
      "so , there are total of 20 numbers .\n",
      "The answer is e, 20\n",
      "\n",
      "Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\n",
      "Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\n",
      "Let's think step by step\n",
      "2a / 100 = 8, a = 400\n",
      "8b / 100 = 2, b = 25\n",
      "c = b / a = 25 / 400 = 1 / 16\n",
      "The answer is a, 1 / 16\n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "probability of first member an english teacher = 3 / 9\n",
      "probability of second member an english teacher = 2 / 8\n",
      "probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\n",
      "The answer is e, 1 / 12\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "r = 7, h = 5\n",
      "2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\n",
      "The answer is b, 528\n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "58 / 4 = 14.5 gallons\n",
      "14.5 * 32 = 464 miles\n",
      "the answer is c, 464\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\".join(prompt_easier_1_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "545d2dfc-35b3-4d29-a605-65f9bacd4ac0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [10:10<00:00,  3.05s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_easier_1_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_easier_1_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "id": "b891388a-a3d9-493f-945c-b199dace327b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 68 ratio 0.3400\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_easier_1, ans_gold_easier_1 =parse_pred_ans('outputs/dev_outputs_coht_easier_1_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35398da-2693-413a-8f79-bd3d7910f73c",
   "metadata": {},
   "source": [
    "## Easy, sentence approx 5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d11d1628-89c4-4b47-b609-75dc5c6658f2",
   "metadata": {},
   "source": [
    "### with \"Let's think\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "968a884d-bc6e-4f53-a6f8-37f40298b544",
   "metadata": {},
   "outputs": [],
   "source": [
    "# easy_idx = np.random.choice(len(train_step_dict[5]), 3 * 8, replace=False)\n",
    "easy_idx = np.array([ 750, 2067, 1126, 1101,  407, 2582, 2191,  442, 2131, 2426,  417,\n",
    "        467,  345,  615, 1351, 2076,   59, 1349,  963, 1831, 1288, 2442,\n",
    "        915,  816])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f69630a2-c792-4444-a389-28c68814199a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 750, 2067, 1126, 1101,  407, 2582, 2191,  442, 2131, 2426,  417,\n",
       "        467,  345,  615, 1351, 2076,   59, 1349,  963, 1831, 1288, 2442,\n",
       "        915,  816])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "easy_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "3565d49a-b2b8-4379-83f4-203e8a6c7735",
   "metadata": {},
   "outputs": [],
   "source": [
    "easy_idx_1 = easy_idx[:8]\n",
    "easy_idx_2 = easy_idx[8: 16]\n",
    "easy_idx_3 = easy_idx[16: ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "04b76c50-553d-4e74-8025-1cda805a5547",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easy_1 = ''\n",
    "\n",
    "for ei in easy_idx_1:\n",
    "    idx = train_step_dict[5][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    # ans_dict = {}\n",
    "    # for a in c.split(' , '):\n",
    "    #     ans_dict[a.split(' ) ')[0]] = a.split(' ) ')[1]\n",
    "    for ac in c.split(','):\n",
    "        idx_, num_ = ac.split(' ) ')\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_easy_1 += 'Question: ' + q + '\\n'\n",
    "    prompt_easy_1 += 'Options: ' + c + '\\n'\n",
    "    prompt_easy_1 += \"Let's think step by step\\n\"\n",
    "    prompt_easy_1 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "e8630880-9163-4a5d-a371-51d89586a0ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: how many books each of volume 250 meter cube can be packed into a crate of volume 4500 meter cube\n",
      "Options: a ) 25 , b ) can not be determined , c ) 10 , d ) 15 , e ) 20\n",
      "Let's think step by step\n",
      "gud question with a simple concept . in geo if we want to insert one shape into another we need to know the dimensions of the two shapes . in above with volume given , we can come up with different shapes , so we cant know the answer for ex : 4500 m 3 can be 450 * 10 or 45 * 100 or just 4500 * 1 we do n ' t know , so we cant calculate answer : b can not be determined \n",
      "\n",
      "Question: a rectangular field is to be fenced on three sides leaving a side of 30 feet uncovered . if the area of the field is 720 sq . feet , how many feet of fencing will be required ?\n",
      "Options: a ) 78 , b ) 40 , c ) 68 , d ) 88 , e ) none\n",
      "Let's think step by step\n",
      "explanation we have : l = 30 ft and lb = 720 sq . ft . so , b = 24 ft . length of fencing = ( l + 2 b ) = ( 30 + 48 ) ft = 78 ft . answer a 78 \n",
      "\n",
      "Question: sahil purchased a machine at rs 13000 , then got it repaired at rs 5000 , then gave its transportation charges rs 1000 . then he sold it with 50 % of profit . at what price he actually sold it .\n",
      "Options: a ) rs . 22000 , b ) rs . 24500 , c ) rs . 26500 , d ) rs . 28500 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : question seems a bit tricky , but it is very simple . just calculate all cost price , then get 150 % of cp . c . p . = 13000 + 5000 + 1000 = 19000 150 % of 19000 = 150 / 100 * 19000 = 28500 option d rs . 28500 \n",
      "\n",
      "Question: a takes twice as much time as b & c takes thrice as much time as b to finish a piece of work . working together they can finish the work in 18 days . the no . of days need for b to do the work alone is ?\n",
      "Options: a ) 30 days , b ) 31 days , c ) 33 days , d ) 41 days , e ) 52 days\n",
      "Let's think step by step\n",
      "let a takes x days to complete the work , then b takes x / 2 days and c takes 3 x / 2 days to complete the work . a , b and c together complete the work in 18 days in 1 day , all they together can do = > 1 / x + 2 / x + 2 / 3 x = 1 / 18 3 + 6 + 2 / 3 x = 1 / 18 3 x 23 x 2 = 18 x = 66 days the number of days b alone takes to finish the work = x / 2 = 33 days c 33 days \n",
      "\n",
      "Question: for the positive integers x , x + 2 , x + 4 , x + 7 , and x + 32 , the mean is how much greater than the median ?\n",
      "Options: a ) 0 , b ) 1 , c ) 2 , d ) 4 , e ) 5\n",
      "Let's think step by step\n",
      "mean = ( x + x + 2 + x + 4 + x + 7 + x + 32 ) / 5 = ( 5 x + 45 ) / 5 = x + 9 median = x + 4 thus mean - median = x + 9 - ( x + 4 ) = 5 answer = e 5\n",
      "\n",
      "Question: find the simple interest on rs 7000 at 50 / 3 % for 9 months\n",
      "Options: a ) rs . 1075 , b ) rs . 975 , c ) rs . 875 , d ) rs . 775 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : s . i . = p × r × t 100 so , by pu ƫ ing the values in the above formula , our result will be . required result = 7000 × 50 × 93 × 12 × 100 = 875 [ please note that we have divided by 12 as we converted 9 months in a year format ] answer : c rs . 875 \n",
      "\n",
      "Question: two trains are moving in opposite directions at 60 km / hr and 90 km / hr . their lengths are 1.10 km and 0.15 km respectively . the time taken by the slower train to cross the faster train in seconds is ?\n",
      "Options: a ) 65 sec , b ) 30 sec , c ) 48 sec , d ) 33 sec , e ) 12 sec\n",
      "Let's think step by step\n",
      "relative speed = 60 + 90 = 150 km / hr . = 150 * 5 / 18 = 125 / 3 m / sec . distance covered = 1.10 + 0.15 = 1.25 km = 1250 m . required time = 1250 * 3 / 125 = 30 sec . answer : b 30 sec \n",
      "\n",
      "Question: a person borrows 7000 for 3 years at 6 % p . a . simple interest . he immediately lends it to another person at 31 % p . a . for 3 years . find his gain in the transaction per year .\n",
      "Options: a ) 1600 , b ) 1700 , c ) 1550 , d ) 1650 , e ) 1750\n",
      "Let's think step by step\n",
      "gain in 3 years = [ ( 7000 ã — 31 ã — 3 / 100 ) â ˆ ’ ( 7000 ã — 6 ã — 3 / 100 ) ] = ( 6510 â € “ 1260 ) = 5250 . â ˆ ´ gain in 1 year = ( 5250 â  „ 3 ) = 1750 answer e 1750\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_easy_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "10ef032c-1c30-4526-a433-5b50a692cb42",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [16:17<00:00,  4.89s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_easy_1.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = prompt_easy_1 + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "813be33e-c2f5-4230-8ba6-0b6742e7aee3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 205 correct 75 ratio 0.3659\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_easy_1, ans_gold_easy_1 = parse_pred_ans('outputs/dev_outputs_coht_easy_1.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13b3b624-6760-40c1-a481-a00d9b76ec30",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### No Let's think"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c22868a-ade6-4cb7-b7ab-18ff8bdac7c1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "ac98247e-ddfc-4596-8ecc-cdc2511d2856",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easy_1_nt = ''\n",
    "\n",
    "for ei in easy_idx_1:\n",
    "    idx = train_step_dict[5][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    # ans_dict = {}\n",
    "    # for a in c.split(' , '):\n",
    "    #     ans_dict[a.split(' ) ')[0]] = a.split(' ) ')[1]\n",
    "    for ac in c.split(','):\n",
    "        idx_, num_ = ac.split(' ) ')\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_easy_1_nt += 'Question: ' + q + '\\n'\n",
    "    prompt_easy_1_nt += 'Options: ' + c + '\\n'\n",
    "    prompt_easy_1_nt += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "eb1afc2d-9629-4775-9ba8-3ea17b778dc8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: how many books each of volume 250 meter cube can be packed into a crate of volume 4500 meter cube\n",
      "Options: a ) 25 , b ) can not be determined , c ) 10 , d ) 15 , e ) 20\n",
      "gud question with a simple concept . in geo if we want to insert one shape into another we need to know the dimensions of the two shapes . in above with volume given , we can come up with different shapes , so we cant know the answer for ex : 4500 m 3 can be 450 * 10 or 45 * 100 or just 4500 * 1 we do n ' t know , so we cant calculate answer : b can not be determined \n",
      "\n",
      "Question: a rectangular field is to be fenced on three sides leaving a side of 30 feet uncovered . if the area of the field is 720 sq . feet , how many feet of fencing will be required ?\n",
      "Options: a ) 78 , b ) 40 , c ) 68 , d ) 88 , e ) none\n",
      "explanation we have : l = 30 ft and lb = 720 sq . ft . so , b = 24 ft . length of fencing = ( l + 2 b ) = ( 30 + 48 ) ft = 78 ft . answer a 78 \n",
      "\n",
      "Question: sahil purchased a machine at rs 13000 , then got it repaired at rs 5000 , then gave its transportation charges rs 1000 . then he sold it with 50 % of profit . at what price he actually sold it .\n",
      "Options: a ) rs . 22000 , b ) rs . 24500 , c ) rs . 26500 , d ) rs . 28500 , e ) none of these\n",
      "explanation : question seems a bit tricky , but it is very simple . just calculate all cost price , then get 150 % of cp . c . p . = 13000 + 5000 + 1000 = 19000 150 % of 19000 = 150 / 100 * 19000 = 28500 option d rs . 28500 \n",
      "\n",
      "Question: a takes twice as much time as b & c takes thrice as much time as b to finish a piece of work . working together they can finish the work in 18 days . the no . of days need for b to do the work alone is ?\n",
      "Options: a ) 30 days , b ) 31 days , c ) 33 days , d ) 41 days , e ) 52 days\n",
      "let a takes x days to complete the work , then b takes x / 2 days and c takes 3 x / 2 days to complete the work . a , b and c together complete the work in 18 days in 1 day , all they together can do = > 1 / x + 2 / x + 2 / 3 x = 1 / 18 3 + 6 + 2 / 3 x = 1 / 18 3 x 23 x 2 = 18 x = 66 days the number of days b alone takes to finish the work = x / 2 = 33 days c 33 days \n",
      "\n",
      "Question: for the positive integers x , x + 2 , x + 4 , x + 7 , and x + 32 , the mean is how much greater than the median ?\n",
      "Options: a ) 0 , b ) 1 , c ) 2 , d ) 4 , e ) 5\n",
      "mean = ( x + x + 2 + x + 4 + x + 7 + x + 32 ) / 5 = ( 5 x + 45 ) / 5 = x + 9 median = x + 4 thus mean - median = x + 9 - ( x + 4 ) = 5 answer = e 5\n",
      "\n",
      "Question: find the simple interest on rs 7000 at 50 / 3 % for 9 months\n",
      "Options: a ) rs . 1075 , b ) rs . 975 , c ) rs . 875 , d ) rs . 775 , e ) none of these\n",
      "explanation : s . i . = p × r × t 100 so , by pu ƫ ing the values in the above formula , our result will be . required result = 7000 × 50 × 93 × 12 × 100 = 875 [ please note that we have divided by 12 as we converted 9 months in a year format ] answer : c rs . 875 \n",
      "\n",
      "Question: two trains are moving in opposite directions at 60 km / hr and 90 km / hr . their lengths are 1.10 km and 0.15 km respectively . the time taken by the slower train to cross the faster train in seconds is ?\n",
      "Options: a ) 65 sec , b ) 30 sec , c ) 48 sec , d ) 33 sec , e ) 12 sec\n",
      "relative speed = 60 + 90 = 150 km / hr . = 150 * 5 / 18 = 125 / 3 m / sec . distance covered = 1.10 + 0.15 = 1.25 km = 1250 m . required time = 1250 * 3 / 125 = 30 sec . answer : b 30 sec \n",
      "\n",
      "Question: a person borrows 7000 for 3 years at 6 % p . a . simple interest . he immediately lends it to another person at 31 % p . a . for 3 years . find his gain in the transaction per year .\n",
      "Options: a ) 1600 , b ) 1700 , c ) 1550 , d ) 1650 , e ) 1750\n",
      "gain in 3 years = [ ( 7000 ã — 31 ã — 3 / 100 ) â ˆ ’ ( 7000 ã — 6 ã — 3 / 100 ) ] = ( 6510 â € “ 1260 ) = 5250 . â ˆ ´ gain in 1 year = ( 5250 â  „ 3 ) = 1750 answer e 1750\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_easy_1_nt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "c9a64ea6-565e-4f0d-be04-2cb0cbdb5dcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|██▋                                                                                                                                 | 4/200 [00:05<04:25,  1.35s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_easy_1_nt.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        \n",
    "        prompt_full = prompt_easy_1 + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        if(i == 5): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "ea8bf250-f3e4-49a4-be0d-2035cf95ed19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 5 correct 3 ratio 0.6000\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_easy_1_nt, ans_gold_easy_1_nt = parse_pred_ans('outputs/dev_outputs_coht_easy_1_nt.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db5f139a-810a-41d5-9175-0f40f6c1f57b",
   "metadata": {},
   "source": [
    "### With linebreak, acc 41.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "5220553c-68fb-4c39-a4b4-704a998eb237",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_easy_1_lb = [\n",
    "\"Question: how many books each of volume 250 meter cube can be packed into a crate of volume 4500 meter cube?\\n\"\n",
    "\"Options: a ) 25 , b ) can not be determined , c ) 10 , d ) 15 , e ) 20\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"if we want to insert one shape into another we need to know the dimensions of the two shapes\\n\",\n",
    "\"in above with volume given , we can come up with different shapes\\n\",\n",
    "\"so we can't know the answer\\n\"\n",
    "\"The answer is b, cannot be determined\\n\\n\",\n",
    "\n",
    "\"Question: a rectangular field is to be fenced on three sides leaving a side of 30 feet uncovered. if the area of the field is 720 sq. feet, how many feet of fencing will be required?\\n\",\n",
    "\"Options: a ) 78 , b ) 40 , c ) 68 , d ) 88 , e ) none\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"we have: l = 30 ft and l * b = 720 sq. ft.\\n\",\n",
    "\"so, b = 24 ft.\\n\",\n",
    "\"length of fencing = (l + 2b) = (30 + 48) ft = 78 ft\\n\", \n",
    "\"The answer is a, 78\\n\\n\"\n",
    "\n",
    "\"Question: sahil purchased a machine at rs 13000, then got it repaired at rs 5000, then gave its transportation charges rs 1000. then he sold it with 50 % of profit. at what price he actually sold it?\",\n",
    "\"Options: a ) rs . 22000 , b ) rs . 24500 , c ) rs . 26500 , d ) rs . 28500 , e ) none of these\\n\",\n",
    "\"Let's think step by step\\n\"\n",
    "\"just calculate all cost price, then get 150 % of the full price\\n\",\n",
    "\"The total cost is 13000 + 5000 + 1000 = 19000\\n\",\n",
    "\"150 % of 19000 = 150 / 100 * 19000 = 28500\\n\", \n",
    "\"The answer is d, rs . 28500\\n\\n\"\n",
    "\n",
    "\"Question: a takes twice as much time as b and c takes thrice as much time as b to finish a piece of work. working together they can finish the work in 18 days. the no. of days need for b to do the work alone is?\"\n",
    "\"Options: a ) 30 days , b ) 31 days , c ) 33 days , d ) 41 days , e ) 52 days\"\n",
    "\"Let's think step by step\\n\",\n",
    "\"let a takes x days to complete the work\\n\", \n",
    "\"then b takes x/2 days and c takes 3x/2 days to complete the work\\n\",\n",
    "\"a, b and c together complete the work in 18 days\\n\",\n",
    "\"all they together can do 1/x + 2/x + 2/3x = 1/18\\n\",\n",
    "\"x = 66 days\\n\",\n",
    "\"the number of days b alone takes to finish the work = x/2 = 33 days\\n\",\n",
    "\"The answer is c, 33 days\\n\\n\",\n",
    "\n",
    "\"Question: for the positive integers x, x + 2, x + 4, x + 7, and x + 32, the mean is how much greater than the median?\\n\",\n",
    "\"Options: a ) 0 , b ) 1 , c ) 2 , d ) 4 , e ) 5\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"mean = (x + x + 2 + x + 4 + x + 7 + x + 32) / 5 = (5x + 45) / 5 = x + 9\\n\",\n",
    "\"median = x + 4\\n\",\n",
    "\"thus mean - median = x + 9 - ( x + 4 ) = 5\\n\",\n",
    "\"The answer is e, 5\\n\\n\",\n",
    "\n",
    "\"Question: find the simple interest on rs 7000 at 50 / 3 % for 9 months\\n\",\n",
    "\"Options: a ) rs . 1075 , b ) rs . 975 , c ) rs . 875 , d ) rs . 775 , e ) none of these\\n\"\n",
    "\"Let's think step by step\\n\",\n",
    "\"s . i . = p × r × t 100 so , by puting the values in the above formula , our result will be\\n\",\n",
    "\"required result = 7000 × 50 × 93 × 12 × 100 = 875\\n\",\n",
    "\"please note that we have divided by 12 as we converted 9 months in a year format\\n\",\n",
    "\"The answer is c, rs . 875\\n\\n\",\n",
    "\n",
    "\"Question: two trains are moving in opposite directions at 60 km / hr and 90 km / hr . their lengths are 1.10 km and 0.15 km respectively . the time taken by the slower train to cross the faster train in seconds is ?\\n\",\n",
    "\"Options: a ) 65 sec , b ) 30 sec , c ) 48 sec , d ) 33 sec , e ) 12 sec\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"relative speed = 60 + 90 = 150 km / hr . = 150 * 5 / 18 = 125 / 3 m / sec.\\n\",\n",
    "\"distance covered = 1.10 + 0.15 = 1.25 km = 1250 m.\\n\",\n",
    "\"required time = 1250 * 3 / 125 = 30 sec.\\n\",\n",
    "\"The answer is b, 30 sec\\n\\n\",\n",
    "\n",
    "\"Question: a person borrows 7000 for 3 years at 6 % p. a. simple interest . he immediately lends it to another person at 31 % p . a . for 3 years . find his gain in the transaction per year?\\n\"\n",
    "\"Options: a ) 1600 , b ) 1700 , c ) 1550 , d ) 1650 , e ) 1750\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"gain in 3 years = [ ( 7000 a — 31 a — 3 / 100 ) a ( 7000 a — 6 a — 3 / 100 ) ] = ( 6510 - 1260 ) = 5250.\\n\",\n",
    "\"gain in 1 year = (5250 a / 3) = 1750\\n\",\n",
    "\"The answer is e, 1750\\n\\n\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b93e722a-de45-40e5-b379-3b7ac007f0ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "step_len = (3 + 3 + 3 + 6 + 3 + 3 + 3 + 3) / 8\n",
    "print(step_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "b88b62db-69ef-4cde-ad63-829e38e096c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: how many books each of volume 250 meter cube can be packed into a crate of volume 4500 meter cube?\n",
      "Options: a ) 25 , b ) can not be determined , c ) 10 , d ) 15 , e ) 20\n",
      "Let's think step by step\n",
      "if we want to insert one shape into another we need to know the dimensions of the two shapes\n",
      "in above with volume given , we can come up with different shapes\n",
      "so we can't know the answer\n",
      "The answer is b, cannot be determined\n",
      "\n",
      "Question: a rectangular field is to be fenced on three sides leaving a side of 30 feet uncovered. if the area of the field is 720 sq. feet, how many feet of fencing will be required?\n",
      "Options: a ) 78 , b ) 40 , c ) 68 , d ) 88 , e ) none\n",
      "Let's think step by step\n",
      "we have: l = 30 ft and l * b = 720 sq. ft.\n",
      "so, b = 24 ft.\n",
      "length of fencing = (l + 2b) = (30 + 48) ft = 78 ft\n",
      "The answer is a, 78\n",
      "\n",
      "Question: sahil purchased a machine at rs 13000, then got it repaired at rs 5000, then gave its transportation charges rs 1000. then he sold it with 50 % of profit. at what price he actually sold it?Options: a ) rs . 22000 , b ) rs . 24500 , c ) rs . 26500 , d ) rs . 28500 , e ) none of these\n",
      "Let's think step by stepjust calculate all cost price, then get 150 % of the full price\n",
      "The total cost is 13000 + 5000 + 1000 = 19000\n",
      "150 % of 19000 = 150 / 100 * 19000 = 28500\n",
      "The answer is d, rs . 28500\n",
      "\n",
      "Question: a takes twice as much time as b and c takes thrice as much time as b to finish a piece of work. working together they can finish the work in 18 days. the no. of days need for b to do the work alone is?Options: a ) 30 days , b ) 31 days , c ) 33 days , d ) 41 days , e ) 52 daysLet's think step by step\n",
      "let a takes x days to complete the work\n",
      "then b takes x/2 days and c takes 3x/2 days to complete the work\n",
      "a, b and c together complete the work in 18 days\n",
      "all they together can do 1/x + 2/x + 2/3x = 1/18\n",
      "x = 66 days\n",
      "the number of days b alone takes to finish the work = x/2 = 33 days\n",
      "The answer is c, 33 days\n",
      "\n",
      "Question: for the positive integers x, x + 2, x + 4, x + 7, and x + 32, the mean is how much greater than the median?\n",
      "Options: a ) 0 , b ) 1 , c ) 2 , d ) 4 , e ) 5\n",
      "Let's think step by step\n",
      "mean = (x + x + 2 + x + 4 + x + 7 + x + 32) / 5 = (5x + 45) / 5 = x + 9\n",
      "median = x + 4\n",
      "thus mean - median = x + 9 - ( x + 4 ) = 5\n",
      "The answer is e, 5\n",
      "\n",
      "Question: find the simple interest on rs 7000 at 50 / 3 % for 9 months\n",
      "Options: a ) rs . 1075 , b ) rs . 975 , c ) rs . 875 , d ) rs . 775 , e ) none of these\n",
      "Let's think step by step\n",
      "s . i . = p × r × t 100 so , by puting the values in the above formula , our result will be\n",
      "required result = 7000 × 50 × 93 × 12 × 100 = 875\n",
      "please note that we have divided by 12 as we converted 9 months in a year format\n",
      "The answer is c, rs . 875\n",
      "\n",
      "Question: two trains are moving in opposite directions at 60 km / hr and 90 km / hr . their lengths are 1.10 km and 0.15 km respectively . the time taken by the slower train to cross the faster train in seconds is ?\n",
      "Options: a ) 65 sec , b ) 30 sec , c ) 48 sec , d ) 33 sec , e ) 12 sec\n",
      "Let's think step by step\n",
      "relative speed = 60 + 90 = 150 km / hr . = 150 * 5 / 18 = 125 / 3 m / sec.\n",
      "distance covered = 1.10 + 0.15 = 1.25 km = 1250 m.\n",
      "required time = 1250 * 3 / 125 = 30 sec.\n",
      "The answer is b, 30 sec\n",
      "\n",
      "Question: a person borrows 7000 for 3 years at 6 % p. a. simple interest . he immediately lends it to another person at 31 % p . a . for 3 years . find his gain in the transaction per year?\n",
      "Options: a ) 1600 , b ) 1700 , c ) 1550 , d ) 1650 , e ) 1750\n",
      "Let's think step by step\n",
      "gain in 3 years = [ ( 7000 a — 31 a — 3 / 100 ) a ( 7000 a — 6 a — 3 / 100 ) ] = ( 6510 - 1260 ) = 5250.\n",
      "gain in 1 year = (5250 a / 3) = 1750\n",
      "The answer is e, 1750\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join(prompt_easy_1_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "4e2db39b-f471-42db-819b-709e79ba02ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [11:37<00:00,  3.49s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_easy_1_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_easy_1_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "33b07e2e-ea73-4b64-a290-68dde3c5ff31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 82 ratio 0.4100\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_easy_1, ans_gold_easy_1 = parse_pred_ans('outputs/dev_outputs_coht_easy_1_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca54ee9a-015e-4a13-a2ed-64927a569a12",
   "metadata": {},
   "source": [
    "## Mid 1, sentence approx 10, avg step 5.75, acc 42.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "1d216eb6-2198-466b-9823-442d672ae85e",
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_idx = np.random.choice(len(train_step_dict[10]), 3 * 8, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e311e71c-b934-498a-96ff-22a956dd371f",
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_idx = np.array([ 94, 287, 223, 282,  89, 411, 213, 379, 202, 339,   1, 338, 313,\n",
    "        88, 279, 323,   6, 300, 131, 499, 198, 232,  24,  46])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "5f667b77-3be8-4f28-bfc8-29b2cf698afd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 94, 287, 223, 282,  89, 411, 213, 379, 202, 339,   1, 338, 313,\n",
       "        88, 279, 323,   6, 300, 131, 499, 198, 232,  24,  46])"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mid_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "586a3ea1-c484-4db4-8f16-c47165f2b3b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_idx_1 = mid_idx[:8]\n",
    "mid_idx_2 = mid_idx[8:]\n",
    "# mid_idx_3 = mid_idx[16: ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "91e3a39f-056b-4f82-b240-c3788a452e4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_mid_1 = ''\n",
    "\n",
    "for ei in mid_idx_1:\n",
    "    idx = train_step_dict[10][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_mid_1 += 'Question: ' + q + '\\n'\n",
    "    prompt_mid_1 += 'Options: ' + c + '\\n'\n",
    "    prompt_mid_1 += \"Let's think step by step\\n\"\n",
    "    prompt_mid_1 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "14c84086-42fd-4ba0-b5ef-119eec4af0f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : let the original average expenditure be rs . x then , 42 ( x - 1 ) - 35 x = 84 ? 7 x = 126 ? x = 18 therefore original expenditure = rs . ( 35 18 ) = rs . 630 . answer : d rs . 630 \n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec . [ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr . let the speed of the train be x km / hr . then , relative speed = ( x - 5 ) km / hr . x - 5 = 72 = = > x = 77 km / hr . answer : c 77 \n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "we can assume the total weight of the mixture = 100 conc of a in the final mixture = 22 let weight of a in the mixture be x . conc given = 20 % = . 2 therefore , weight of b = 100 - x . conc given = 30 % = . 3 now , acc to the problem , . 2 x + . 3 ( 100 - x ) = 22 solving , we get x = 80 . since we assumed the weight of the mixture = 100 , therefore presence of a in the mixture = 80 % . answer : c 80 % \n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : sp of 1 metre cloth = 8200 / 40 = rs . 205 . cp of 1 metre cloth = rs . 205 – 35 = rs . 170 cp on 40 metres = 170 x 40 = rs . 6800 profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400 . answer : option d rs . 1400 \n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "x < y < z to find the least possible value for z - x ; we need to find the values for z and x that can be closest to each other . if x is some even number , then what could be minimum possible odd z . if x is some even number y - x > 5 ; y > x + 5 ; minimum value for y = x + 5 + 2 = x + 7 [ note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2 ] minimum value for z = y + 2 = x + 7 + 2 = x + 9 [ note : z = y + 2 because both z and y are odd . difference between two odd numbers is 2 ] s = z - x = x + 9 - x = 9 ans : d 9 \n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8 c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25 difference = ( 6367.25 - 6364.8 ) = rs . 2.45 . answer : e s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : let a , b , c represent their respective weights . then , we have : a + b + c = ( 45 x 3 ) = 135 … . ( i ) a + b = ( 40 x 2 ) = 80 … . ( ii ) b + c = ( 45 x 2 ) = 90 … . ( iii ) adding ( ii ) and ( iii ) , we get : a + 2 b + c = 170 … . ( iv ) subtracting ( i ) from ( iv ) , we get : b = 35 . b ’ s weight = 35 kg . answer : d 35 kg \n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "explanation : the simple interest for the first year is 10200 / 2 is rs . 5100 and compound interest for first year also is rs . 5100 . the compound interest for second year on rs . 5100 for one year so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30 % p . a . so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000 answer : b rs . 17000 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_mid_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "ed33ffef-e8de-4c45-b367-ceb4059d84a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_mid_1_lb = [\n",
    "\"Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\\n\",\n",
    "\"Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"let the original average expenditure be rs . x\\n\",\n",
    "\"then , 42 ( x - 1 ) - 35 x = 84\\,\",\n",
    "\"7 x = 126\\n\",\n",
    "\"x = 18\\n\",\n",
    "\"therefore original expenditure = rs . ( 35 * 18 ) = rs . 630.\\n\",\n",
    "\"The answer is d, rs . 630\\n\\n\",\n",
    "\n",
    "\"Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\\n\",\n",
    "\"Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec.\\n\",\n",
    "\"[ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr.\\n\",\n",
    "\"let the speed of the train be x km / hr.\\n\",\n",
    "\"then , relative speed = ( x - 5 ) km / hr.\\n\",\n",
    "\"x - 5 = 72, x = 77 km / hr .\\n\", \n",
    "\"The answer is c, 77\\n\\n\",\n",
    "\n",
    "\"Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\\n\"\n",
    "\"Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"we can assume the total weight of the mixture = 100\\n\",\n",
    "\"conc of a in the final mixture = 22\\n\",\n",
    "\"let weight of a in the mixture be x.\\n\",\n",
    "\"conc given = 20% = 0.2\\n\",\n",
    "\"therefore , weight of b = 100 - x.\\n\",\n",
    "\"conc given = 30% = 0.3\\n\",\n",
    "\"now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22\\n\",\n",
    "\"solving , we get x = 80\\n\",\n",
    "\"since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%.\\n\",\n",
    "\"The answer is c, 80%\\n\\n\"\n",
    "\n",
    "\"Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\\n\",\n",
    "\"Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"price of 1 metre cloth = 8200 / 40 = rs 205\\n\",\n",
    "\"cost of 1 metre cloth = rs 205 – 35 = rs 170\\n\",\n",
    "\"cost on 40 metres = 170 x 40 = rs . 6800\\n\",\n",
    "\"profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400\\n\",\n",
    "\"The answer is d, rs . 1400\\n\\n\"\n",
    "\n",
    "\"Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\\n\"\n",
    "\"Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"We know x < y < z\\n\"\n",
    "\"to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other.\\n\"\n",
    "\"if x is some even number, then what could be minimum possible odd z.\\n\",\n",
    "\"if x is some even number, y - x > 5 ; y > x + 5\\n\",\n",
    "\"minimum value for y = x + 5 + 2 = x + 7\\n\",\n",
    "\"(note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2)\\n\",\n",
    "\"minimum value for z = y + 2 = x + 7 + 2 = x + 9\\n\",\n",
    "\"(note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2)\\n\",\n",
    "\"s = z - x = x + 9 - x = 9\\n\",\n",
    "\"The answer is d, 9\\n\\n\",\n",
    "\n",
    "\"Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\\n\"\n",
    "\"Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8\\n\",\n",
    "\"c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25\\n\",\n",
    "\"difference = ( 6367.25 - 6364.8 ) = rs . 2.45.\\n\",\n",
    "\"The answer is e, s . 2.45\\n\\n\",\n",
    "\n",
    "\"Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\\n\",\n",
    "\"Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\\n\"\n",
    "\"Let's think step by step\\n\",\n",
    "\"let a , b , c represent their respective weights.\\n\",\n",
    "\"then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i )\\n\",\n",
    "\"a + b = ( 40 x 2 ) = 80 … ( ii )\\n\",\n",
    "\"b + c = ( 45 x 2 ) = 90 … ( iii )\\n\",\n",
    "\"adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv )\\n\",\n",
    "\"subtracting ( i ) from ( iv ), we get : b = 35.\\n\",\n",
    "\"b’ s weight = 35 kg .\\n\",\n",
    "\"The answer is d 35 kg\\n\\n\"\n",
    "\n",
    "\"Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\\n\"\n",
    "\"Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"the simple interest for the first year is 10200 / 2 is rs . 5100\\n\",\n",
    "\"and compound interest for first year also is rs . 5100.\\n\",\n",
    "\"the compound interest for second year on rs . 5100 for one year\\n\",\n",
    "\"so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p\\n\",\n",
    "\"so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000\\n\",\n",
    "\"The answer is b, rs . 17000\\n\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ae3d865-b9a0-43d6-8b76-b5a01bc1553e",
   "metadata": {},
   "outputs": [],
   "source": [
    "step_len = (4 + 5 + 9 + 4 + 9 + 3 + 7 + 5) / 8\n",
    "print(step_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "a0b9e029-ddfa-4384-830a-7b349fd8c939",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "let the original average expenditure be rs . x\n",
      "then , 42 ( x - 1 ) - 35 x = 84\\,7 x = 126\n",
      "x = 18\n",
      "therefore original expenditure = rs . ( 35 * 18 ) = rs . 630.\n",
      "The answer is d, rs . 630\n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec.\n",
      "[ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr.\n",
      "let the speed of the train be x km / hr.\n",
      "then , relative speed = ( x - 5 ) km / hr.\n",
      "x - 5 = 72, x = 77 km / hr .\n",
      "The answer is c, 77\n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "we can assume the total weight of the mixture = 100\n",
      "conc of a in the final mixture = 22\n",
      "let weight of a in the mixture be x.\n",
      "conc given = 20% = 0.2\n",
      "therefore , weight of b = 100 - x.\n",
      "conc given = 30% = 0.3\n",
      "now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22\n",
      "solving , we get x = 80\n",
      "since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%.\n",
      "The answer is c, 80%\n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "price of 1 metre cloth = 8200 / 40 = rs 205\n",
      "cost of 1 metre cloth = rs 205 – 35 = rs 170\n",
      "cost on 40 metres = 170 x 40 = rs . 6800\n",
      "profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400\n",
      "The answer is d, rs . 1400\n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "We know x < y < z\n",
      "to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other.\n",
      "if x is some even number, then what could be minimum possible odd z.\n",
      "if x is some even number, y - x > 5 ; y > x + 5\n",
      "minimum value for y = x + 5 + 2 = x + 7\n",
      "(note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2)\n",
      "minimum value for z = y + 2 = x + 7 + 2 = x + 9\n",
      "(note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2)\n",
      "s = z - x = x + 9 - x = 9\n",
      "The answer is d, 9\n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8\n",
      "c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25\n",
      "difference = ( 6367.25 - 6364.8 ) = rs . 2.45.\n",
      "The answer is e, s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of theseLet's think step by step\n",
      "let a , b , c represent their respective weights.\n",
      "then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i )\n",
      "a + b = ( 40 x 2 ) = 80 … ( ii )\n",
      "b + c = ( 45 x 2 ) = 90 … ( iii )\n",
      "adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv )\n",
      "subtracting ( i ) from ( iv ), we get : b = 35.\n",
      "b’ s weight = 35 kg .\n",
      "The answer is d 35 kg\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is rs . 5100\n",
      "and compound interest for first year also is rs . 5100.\n",
      "the compound interest for second year on rs . 5100 for one year\n",
      "so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p\n",
      "so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000\n",
      "The answer is b, rs . 17000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\".join(prompt_mid_1_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "id": "9a039be1-66ea-4264-af6a-d370ff013cbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [16:11<00:00,  4.86s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_mid_1_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_mid_1_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "72d81f3c-2b30-4452-8e9a-9726a3a53562",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 85 ratio 0.4250\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_mid_1, ans_gold_mid_1 =parse_pred_ans('outputs/dev_outputs_coht_mid_1_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2d78b49-603f-4952-ad99-694dc0fd3346",
   "metadata": {},
   "source": [
    "## Mid 2, avg step 4.75, acc 41.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "id": "9732d92b-9fd7-4a9c-9b7a-aced5ad19fc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_mid_2 = ''\n",
    "\n",
    "for ei in mid_idx_2:\n",
    "    idx = train_step_dict[10][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_mid_2 += 'Question: ' + q + '\\n'\n",
    "    prompt_mid_2 += 'Options: ' + c + '\\n'\n",
    "    prompt_mid_2 += \"Let's think step by step\\n\"\n",
    "    prompt_mid_2 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "36f721ec-4a1c-4b22-ad6e-201cceb3b4f6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a goods train runs at the speed of 72 kmph and crosses a 350 m long platform in 26 seconds . what is the length of the goods train ?\n",
      "Options: a ) 230 m , b ) 140 m , c ) 160 m , d ) 170 m , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : speed = [ 72 x ( 5 / 18 ) ] m / sec = 20 m / sec . time = 26 sec . let the length of the train be x metres . then , [ ( x + 350 ) / 26 ] = 20 = > x + 350 = 520 = > x = 170 . answer : d 170 m \n",
      "\n",
      "Question: machine a and machine b are each used to manufacture 660 sprockets . it takes machine a 10 hours longer to produce 660 sprockets than machine b . machine b produces 10 percent more sprockets per hour than machine a . how many sprockets per hour does machine a produces ?\n",
      "Options: a ) 6 , b ) 6.6 , c ) 60 , d ) 100 , e ) 110\n",
      "Let's think step by step\n",
      "b takes x hours a takes x + 10 hours rate of a = 660 / x + 10 rate of b = 660 / x thus , 660 / x = ( 660 / x + 10 ) * 1.10 x = 100 so b = 100 a = 110 sprockets per hour 660 / 110 = 6 answer : a 6 \n",
      "\n",
      "Question: a certain list consists of 21 different numbers . if n is in the list and n is 7 times the average ( arithmetic mean ) of the other 20 numbers in the list , then n is what fraction of the sum of the 21 numbers in the list ?\n",
      "Options: a ) 7 / 27 , b ) 1 / 6 , c ) 1 / 5 , d ) 4 / 21 , e ) 5 / 21\n",
      "Let's think step by step\n",
      "series : a 1 , a 2 . . . . a 20 , n sum of a 1 + a 2 + . . . + a 20 = 20 * x ( x = average ) so , n = 7 * x hence , a 1 + a 2 + . . + a 20 + n = 27 x so , the fraction asked = 7 x / 27 x = 7 / 27 answer a 7 / 27 \n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum .\n",
      "Options: a ) 33498 , b ) 17000 , c ) 2665 , d ) 28876 , e ) 313\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is rs . 5100 and compound interest for first year also is rs . 5100 . the compound interest for second year on rs . 5100 for one year so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30 % p . a . so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000 answer : b 17000 \n",
      "\n",
      "Question: find the area , diameter = 9 m .\n",
      "Options: a ) 63.00 square meter , b ) 63.64 square meter , c ) 63.08 square meter , d ) 63.24 square meter , e ) 63.43 square meter\n",
      "Let's think step by step\n",
      "diameter = 9 meter . radius = diameter / 2 . = 9 / 2 . = 4.5 meter . area of a circle = ï € r 2 . here , pi ( ï € ) = 3.14 meter , radius ( r ) = 4.5 . area of a circle = 3.14 ã — 4.5 ã — 4.5 . = 3.14 ã — 20.25 . = 63.64 m 2 . answer : b 63.64 square meter \n",
      "\n",
      "Question: car q and car y traveled the same 80 - mile route . if car q took 2 hours and car y traveled at an average speed that was 50 percent faster than the average speed of car q , how many hours did it take car y to travel the route ?\n",
      "Options: a ) 2 / 3 , b ) 1 , c ) 4 / 3 , d ) 8 / 5 , e ) 3\n",
      "Let's think step by step\n",
      "the speed of car q is ( distance ) / ( time ) = 80 / 2 = 40 miles per hour . the speed of car y = 3 / 2 * 40 = 60 miles per hour - - > ( time ) = ( distance ) / ( speed ) = 80 / 60 = 4 / 3 hours . answer : c . or : to cover the same distance at 3 / 2 as fast rate 2 / 3 as much time is needed - - > ( time ) * 2 / 3 = 2 * 2 / 3 = 4 / 3 hours . answer : c . 4 / 3 \n",
      "\n",
      "Question: 24 oz of juice p and 25 oz of juice q are mixed to make smothies x and y . the ratio of p to q in smothie x is 4 is to 1 and that in y is 1 is to 5 . how many ounces of juice p are contained in the smothie x ?\n",
      "Options: a ) 5 , b ) 10 , c ) 15 , d ) 20 , e ) 25\n",
      "Let's think step by step\n",
      "easy way to solve this question is start from the answer and then conform the information provided in the question . we can start from option d i . e 20 . . . as a quantity of juice p in x because it is the only one option that gets divided by 4 is 20 . . . since in the x the juice p to q ratio is 4 : 1 this gives us that quantity of juice p in x = 20 therefore quantity of juice v will be 5 . . . hence ratio = 4 : 1 this will lead to quantity of juice p in x = 4 and quantity of juice v = 20 . . . hence ratio 1 : 5 if we calculate total juice p = 24 and total of juice q = 25 it fits because totals are same as what mentioned in the question . . . thus ans is d 20 \n",
      "\n",
      "Question: a room 5 m 55 cm long and 3 m 74 cm broad is to be paved with square tiles . find the least number of square tiles required to cover the floor\n",
      "Options: a ) 176 , b ) 170 , c ) 160 , d ) 179 , e ) 174\n",
      "Let's think step by step\n",
      "let x and y be the sides of the rectangle . then , correct area = xy . area of the room = ( 544 x 374 ) cm 2 . size of largest square tile = h . c . f . of 544 cm and 374 cm = 34 cm . area of 1 tile = ( 34 x 34 ) cm 2 . number of tiles required = ( 544 * 374 ) / ( 34 * 34 ) = 176 answer a 176 176 \n",
      "\n",
      "Question: average expenditure of a person for the first 3 days of a week is rs . 350 and for the next 4 days is rs . 420 . average expenditure of the man for the whole week is :\n",
      "Options: a ) 350 , b ) 370 , c ) 390 , d ) 430 , e ) none\n",
      "Let's think step by step\n",
      "explanation : assumed mean = rs . 350 total excess than assumed mean = 4 × ( rs . 420 - rs . 350 ) = rs . 280 therefore , increase in average expenditure = rs . 280 / 7 = rs . 40 therefore , average expenditure for 7 days = rs . 350 + rs . 40 = rs . 390 correct option : c 390 \n",
      "\n",
      "Question: the length of the bridge , which a train 200 metres long and travelling at 60 km / hr can cross in 30 seconds , is ?\n",
      "Options: a ) 720 , b ) 470 , c ) 270 , d ) 370 m . , e ) 570\n",
      "Let's think step by step\n",
      "speed = [ 60 x 5 / 18 ] m / sec = [ 50 / 3 ] m / sec time = 30 sec let the length of bridge be x metres . then , ( 130 + x ) / 30 = 50 / 3 = > ( 130 + x ) / 30 = 370 = > x = 370 m . answer : d 370 m . \n",
      "\n",
      "Question: the circumference of the front wheel of a cart is 30 ft long and that of the back wheel is 33 ft long . what is the distance traveled by the cart , when the front wheel has done five more revolutions than the rear wheel ?\n",
      "Options: ['a ) 20 ft', 'b ) 25 ft', 'c ) 750 ft', 'd ) 900 ft', 'e ) 1650 ft']\n",
      "Let's think step by step\n",
      "point to note : both the wheels would have traveled the same distance . now consider , no . of revolutions made by back wheel as x , which implies that the number of revolutions made by the front wheel is ( x + 5 ) . equating the distance traveled by front wheel to back wheel : ( x + 5 ) * 30 = x * 33 . ( formula for calculating the distance traveled by each wheel is : # of revolutions * circumference . ) solving this eqn . gives x = 50 . sub x = 50 either in ( x + 5 ) * 30 or in x * 33 to get the distance , which is 1650 . so the correct choice is e . 1650 ft']\n",
      "\n",
      "Question: if a randomly selected non - negative single digit integer is added to { 2 , 3 , 5 , 8 } . what is the probability that the median of the set will increase but the range still remains the same ?\n",
      "Options: a ) 0.2 , b ) 0.3 , c ) 0.4 , d ) 0.5 , e ) 0.6\n",
      "Let's think step by step\n",
      "we are selecting from non - negative single digit integers , so from { 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 } . these 10 digits represent the total number of outcomes . hence , the total number of outcomes is 10 . we need to find the probability that the median of the set will increase but the range still remains the same . the median of the set is ( 3 + 5 ) / 2 = 4 , thus the number selected must be 5 or greater . for the range to remain the same , the number must be between 2 and 8 inclusive . to satisfy both conditions , the number selected must be 5 , 6 , 7 , or 8 . the probability is 4 / 10 = 0.4 the answer is c . 0.4 \n",
      "\n",
      "Question: a person borrows rs . 5000 for 2 years at 5 % p . a . simple interest . he immediately lends it to another person at 61 ⁄ 4 % p . a for 2 years . find his gain in the transaction per year .\n",
      "Options: a ) 62.5 , b ) 150 , c ) 225 , d ) 112.5 , e ) 212.5\n",
      "Let's think step by step\n",
      "explanation : the person borrows rs . 5000 for 2 years at 5 % p . a . simple interest simple interest that he needs to pay = prt / 100 = 5000 × 5 × 2 / 100 = 500 he also lends it at 6 1 ⁄ 4 % p . a for 2 years simple interest that he gets = prt / 100 = 5000 × 25 / 4 × 2 / 100 = 625 his overall gain in 2 years = rs . 625 - rs . 500 = rs . 125 his overall gain in 1 year = 125 / 2 = rs . 62.5 answer : option a 62.5 \n",
      "\n",
      "Question: pascal has 96 miles remaining to complete his cycling trip . if he reduced his current speed by 4 miles per hour , the remainder of the trip would take him 16 hours longer than it would if he increased his speed by 50 % . what is his current speed o ?\n",
      "Options: a ) 6 , b ) 8 , c ) 10 , d ) 12 , e ) 16\n",
      "Let's think step by step\n",
      "let the current speed be x miles per hour . time taken if speed is 50 % faster ( i . e . 3 x / 2 = 1.5 x ) = 96 / 1.5 x time taken if speed is reduced by 4 miles / hr ( i . e . ( x - 4 ) ) = 96 / ( x - 4 ) as per question , 96 / ( x - 4 ) - 96 / 1.5 x = 16 solving this o we get x = 8 . b . 8 \n",
      "\n",
      "Question: the average weight of a , b and c is 30 kg . if the average weight of a and b be 25 kg and that of b and c be 28 kg , then the weight of b is :\n",
      "Options: a ) 14 kg , b ) 15 kg , c ) 18 kg , d ) 19 kg , e ) 16 kg\n",
      "Let's think step by step\n",
      "explanation let a , b , c represent their respective weights . then , we have : a + b + c = ( 30 x 3 ) = 90 â € ¦ . ( i ) a + b = ( 25 x 2 ) = 50 â € ¦ . ( ii ) b + c = ( 28 x 2 ) = 56 â € ¦ . ( iii ) adding ( ii ) and ( iii ) , we get : a + 2 b + c = 106 â € ¦ . ( iv ) subtracting ( i ) from ( iv ) , we get : b = 16 . b â € ™ s weight = 16 kg . answer e 16 kg\n",
      "\n",
      "Question: a grocer has a sale of rs . 5400 , rs . 9000 , rs . 6300 , rs . 7200 and rs . 4500 for 5 consecutive months . how much sale must he have in the sixth month so that he gets an average sale of rs . 5600 ?\n",
      "Options: a ) s . 1200 , b ) s . 5400 , c ) s . 5400 , d ) s . 4999 , e ) s . 5000\n",
      "Let's think step by step\n",
      "total sale for 5 months = rs . ( 5400 + 9000 + 6300 + 7200 + 4500 ) = rs . 32400 . required sale = rs . [ ( 5600 x 6 ) - 32400 ] = rs . ( 33600 - 32400 ) = rs . 1200 . option a s . 1200 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_mid_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "dab32407-3a3b-4189-8a93-ba90f5ed6215",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_mid_2_lb = [\n",
    "\"Question: a goods train runs at the speed of 72 kmph and crosses a 350 m long platform in 26 seconds . what is the length of the goods train ?\\n\"\n",
    "\"Options: a ) 230 m , b ) 140 m , c ) 160 m , d ) 170 m , e ) none of these\\n\",\n",
    "\"Let's think step by step\\n\"\n",
    "\"speed = [ 72 x ( 5 / 18 ) ] m / sec = 20 m / sec.\\n\",\n",
    "\"time = 26 sec.\\n\",\n",
    "\"let the length of the train be x metres.\\n\",\n",
    "\"then [ ( x + 350 ) / 26 ] = 20\\n\",\n",
    "\"x + 350 = 520\\n\",\n",
    "\"x = 170.\\n\"\n",
    "\"The answer is d, 170 m\\n\\n\"\n",
    "\n",
    "\"Question: machine a and machine b are each used to manufacture 660 sprockets . it takes machine a 10 hours longer to produce 660 sprockets than machine b . machine b produces 10 percent more sprockets per hour than machine a . how many sprockets per hour does machine a produces ?\\n\"\n",
    "\"Options: a ) 6 , b ) 6.6 , c ) 60 , d ) 100 , e ) 110\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"b takes x hours, a takes x + 10 hours\\n\",\n",
    "\"rate of a = 660 / x + 10, rate of b = 660 / x\\n\",\n",
    "\"thus , 660 / x = ( 660 / x + 10 ) * 1.10\\n\",\n",
    "\"x = 100, so b = 100, a = 110\\n\",\n",
    "\"sprockets per hour 660 / 110 = 6\\n\",\n",
    "\"The answer is a, 6\\n\\n\"\n",
    "\n",
    "\"Question: a certain list consists of 21 different numbers . if n is in the list and n is 7 times the average ( arithmetic mean ) of the other 20 numbers in the list , then n is what fraction of the sum of the 21 numbers in the list ?\\n\"\n",
    "\"Options: a ) 7 / 27 , b ) 1 / 6 , c ) 1 / 5 , d ) 4 / 21 , e ) 5 / 21\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"series: a1, a2 ... a20\\n\",\n",
    "\"sum of a1 + a2 + . . . + a20 = 20 * x ( x = average )\\n\",\n",
    "\"so , n = 7 * x\\n\",\n",
    "\"hence, a1 + a2 + . . + a20 + n = 27x\\n\",\n",
    "\"so , the fraction asked = 7 x / 27 x = 7 / 27\\n\",\n",
    "\"The answer is a, 7 / 27\\n\\n\"\n",
    "\n",
    "\"Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum .\\n\"\n",
    "\"Options: a ) 33498 , b ) 17000 , c ) 2665 , d ) 28876 , e ) 313\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"the simple interest for the first year is 10200 / 2 is 5100\\n\",\n",
    "\"and compound interest for first year also is 5100.\\n\",\n",
    "\"the compound interest for second year on 5100 for one year\\n\",\n",
    "\"so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p\\n\",\n",
    "\"so p = ( 100 * 10200 ) / ( 30 * 2 ) = 17000\\n\",\n",
    "\"The answer is b, 17000\\n\\n\"\n",
    "\n",
    "\"Question: find the area , diameter = 9 m .\\n\"\n",
    "\"Options: a ) 63.00 square meter , b ) 63.64 square meter , c ) 63.08 square meter , d ) 63.24 square meter , e ) 63.43 square meter\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"diameter = 9 meter.\\n\", \n",
    "\"radius = diameter / 2 . = 9 / 2 . = 4.5 meter.\\n\",\n",
    "\"area of a circle = pi * r^2.\\n\",\n",
    "\"here , pi = 3.14 meter, radius ( r ) = 4.5.\\n\",\n",
    "\"area of a circle = 3.14 * 4.5 * 4.5 = 3.14 * 20.25 = 63.64 m^2\\n\",\n",
    "\"The answer is b, 63.64 square meter\\n\\n\"\n",
    "\n",
    "\"Question: car q and car y traveled the same 80 - mile route . if car q took 2 hours and car y traveled at an average speed that was 50 percent faster than the average speed of car q , how many hours did it take car y to travel the route ?\\n\"\n",
    "\"Options: a ) 2 / 3 , b ) 1 , c ) 4 / 3 , d ) 8 / 5 , e ) 3\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"the speed of car q is ( distance ) / ( time ) = 80 / 2 = 40 miles per hour.\\n\",\n",
    "\"the speed of car y = 3 / 2 * 40 = 60 miles per hour\\n\",\n",
    "\"( time ) = ( distance ) / ( speed ) = 80 / 60 = 4 / 3 hours.\\n\",\n",
    "\"The answer is c, 4 / 3\\n\\n\"\n",
    "\n",
    "\"Question: a room 5 m 55 cm long and 3 m 74 cm broad is to be paved with square tiles . find the least number of square tiles required to cover the floor\\n\",\n",
    "\"Options: a ) 176 , b ) 170 , c ) 160 , d ) 179 , e ) 174\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"let x and y be the sides of the rectangle.\\n\",\n",
    "\"then , correct area = xy.\\n\",\n",
    "\"area of the room = 544 x 374 cm^2.\\n\",\n",
    "\"size of largest square tile = greatest common divisor of 544 cm and 374 cm = 34 cm\\n\",\n",
    "\"area of 1 tile = ( 34 x 34 ) cm^2.\\n\",\n",
    "\"number of tiles required = ( 544 * 374 ) / ( 34 * 34 ) = 176\\n\",\n",
    "\"The answer a, 176\\n\\n\",\n",
    "    \n",
    "\"Question: pascal has 96 miles remaining to complete his cycling trip . if he reduced his current speed by 4 miles per hour , the remainder of the trip would take him 16 hours longer than it would if he increased his speed by 50 % . what is his current speed o ?\\n\"\n",
    "\"Options: a ) 6 , b ) 8 , c ) 10 , d ) 12 , e ) 16\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"let the current speed be x miles per hour time taken\\n\",\n",
    "\"if speed is 50 % faster (i.e . 3x / 2 = 1.5 x) = 96 / 1.5 x time taken\\n\",\n",
    "\"if speed is reduced by 4 miles / hr (i.e. ( x - 4 ) ) = 96 / ( x - 4 )\\n\",\n",
    "\"as per question, 96 / ( x - 4 ) - 96 / 1.5 x = 16\\n\",\n",
    "\"solving this we get x = 8.\\n\",\n",
    "\"The answer is b, 8\\n\\n\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d235940b-17c5-43fb-952e-507805437d5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "step_len = (6 + 5 + 5 + 5 +  4 + 3 + 5 + 5) / 8\n",
    "print(step_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "a3f13749-70be-4aa6-baa7-056a13318501",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a goods train runs at the speed of 72 kmph and crosses a 350 m long platform in 26 seconds . what is the length of the goods train ?\n",
      "Options: a ) 230 m , b ) 140 m , c ) 160 m , d ) 170 m , e ) none of these\n",
      "Let's think step by step\n",
      "speed = [ 72 x ( 5 / 18 ) ] m / sec = 20 m / sec.\n",
      "time = 26 sec.\n",
      "let the length of the train be x metres.\n",
      "then [ ( x + 350 ) / 26 ] = 20\n",
      "x + 350 = 520\n",
      "x = 170.\n",
      "The answer is d, 170 m\n",
      "\n",
      "Question: machine a and machine b are each used to manufacture 660 sprockets . it takes machine a 10 hours longer to produce 660 sprockets than machine b . machine b produces 10 percent more sprockets per hour than machine a . how many sprockets per hour does machine a produces ?\n",
      "Options: a ) 6 , b ) 6.6 , c ) 60 , d ) 100 , e ) 110\n",
      "Let's think step by step\n",
      "b takes x hours, a takes x + 10 hours\n",
      "rate of a = 660 / x + 10, rate of b = 660 / x\n",
      "thus , 660 / x = ( 660 / x + 10 ) * 1.10\n",
      "x = 100, so b = 100, a = 110\n",
      "sprockets per hour 660 / 110 = 6\n",
      "The answer is a, 6\n",
      "\n",
      "Question: a certain list consists of 21 different numbers . if n is in the list and n is 7 times the average ( arithmetic mean ) of the other 20 numbers in the list , then n is what fraction of the sum of the 21 numbers in the list ?\n",
      "Options: a ) 7 / 27 , b ) 1 / 6 , c ) 1 / 5 , d ) 4 / 21 , e ) 5 / 21\n",
      "Let's think step by step\n",
      "series: a1, a2 ... a20\n",
      "sum of a1 + a2 + . . . + a20 = 20 * x ( x = average )\n",
      "so , n = 7 * x\n",
      "hence, a1 + a2 + . . + a20 + n = 27x\n",
      "so , the fraction asked = 7 x / 27 x = 7 / 27\n",
      "The answer is a, 7 / 27\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum .\n",
      "Options: a ) 33498 , b ) 17000 , c ) 2665 , d ) 28876 , e ) 313\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is 5100\n",
      "and compound interest for first year also is 5100.\n",
      "the compound interest for second year on 5100 for one year\n",
      "so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p\n",
      "so p = ( 100 * 10200 ) / ( 30 * 2 ) = 17000\n",
      "The answer is b, 17000\n",
      "\n",
      "Question: find the area , diameter = 9 m .\n",
      "Options: a ) 63.00 square meter , b ) 63.64 square meter , c ) 63.08 square meter , d ) 63.24 square meter , e ) 63.43 square meter\n",
      "Let's think step by step\n",
      "diameter = 9 meter.\n",
      "radius = diameter / 2 . = 9 / 2 . = 4.5 meter.\n",
      "area of a circle = pi * r^2.\n",
      "here , pi = 3.14 meter, radius ( r ) = 4.5.\n",
      "area of a circle = 3.14 * 4.5 * 4.5 = 3.14 * 20.25 = 63.64 m^2\n",
      "The answer is b, 63.64 square meter\n",
      "\n",
      "Question: car q and car y traveled the same 80 - mile route . if car q took 2 hours and car y traveled at an average speed that was 50 percent faster than the average speed of car q , how many hours did it take car y to travel the route ?\n",
      "Options: a ) 2 / 3 , b ) 1 , c ) 4 / 3 , d ) 8 / 5 , e ) 3\n",
      "Let's think step by step\n",
      "the speed of car q is ( distance ) / ( time ) = 80 / 2 = 40 miles per hour.\n",
      "the speed of car y = 3 / 2 * 40 = 60 miles per hour\n",
      "( time ) = ( distance ) / ( speed ) = 80 / 60 = 4 / 3 hours.\n",
      "The answer is c, 4 / 3\n",
      "\n",
      "Question: a room 5 m 55 cm long and 3 m 74 cm broad is to be paved with square tiles . find the least number of square tiles required to cover the floor\n",
      "Options: a ) 176 , b ) 170 , c ) 160 , d ) 179 , e ) 174\n",
      "Let's think step by step\n",
      "let x and y be the sides of the rectangle.\n",
      "then , correct area = xy.\n",
      "area of the room = 544 x 374 cm^2.\n",
      "size of largest square tile = greatest common divisor of 544 cm and 374 cm = 34 cm\n",
      "area of 1 tile = ( 34 x 34 ) cm^2.\n",
      "number of tiles required = ( 544 * 374 ) / ( 34 * 34 ) = 176\n",
      "The answer a, 176\n",
      "\n",
      "Question: pascal has 96 miles remaining to complete his cycling trip . if he reduced his current speed by 4 miles per hour , the remainder of the trip would take him 16 hours longer than it would if he increased his speed by 50 % . what is his current speed o ?\n",
      "Options: a ) 6 , b ) 8 , c ) 10 , d ) 12 , e ) 16\n",
      "Let's think step by step\n",
      "let the current speed be x miles per hour time taken\n",
      "if speed is 50 % faster (i.e . 3x / 2 = 1.5 x) = 96 / 1.5 x time taken\n",
      "if speed is reduced by 4 miles / hr (i.e. ( x - 4 ) ) = 96 / ( x - 4 )\n",
      "as per question, 96 / ( x - 4 ) - 96 / 1.5 x = 16\n",
      "solving this we get x = 8.\n",
      "The answer is b, 8\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join(prompt_mid_2_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "id": "fcba3bff-8f6b-4044-a0e7-aaca87fa6b6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [14:37<00:00,  4.39s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_mid_2_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_mid_2_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "id": "930e7c28-dd60-4377-aacc-472c5c8dd657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 82 ratio 0.4100\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_mid_2, ans_gold_mid_2 =parse_pred_ans('outputs/dev_outputs_coht_mid_2_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e66bfac-6446-4364-a919-811daf862cf8",
   "metadata": {},
   "source": [
    "## Hard 1, sentence approx 18, acc 31.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "ea3f4aff-25f1-4d39-b097-f66f0bedc453",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hard_idx = np.random.choice(len(train_step_dict[18]), 3 * 8, replace=False)\n",
    "hard_idx = np.array([26,  7, 23, 15, 40,  5, 30, 54,  4, 44, 33, 56, 11, 50, 39, 31, 51,\n",
    "       45, 10, 36, 13, 47, 12, 38])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "a773aae3-49c3-4e6f-939f-9cdb44adf863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([26,  7, 23, 15, 40,  5, 30, 54,  4, 44, 33, 56, 11, 50, 39, 31, 51,\n",
       "       45, 10, 36, 13, 47, 12, 38])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hard_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "cbd34110-11cb-4b23-a16d-98cb9543c8cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "hard_idx_1 = hard_idx[:8]\n",
    "hard_idx_2 = hard_idx[8: 16]\n",
    "hard_idx_3 = hard_idx[16: ]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94250bbc-d388-4d8f-a23b-a2689a52ea29",
   "metadata": {},
   "source": [
    "### Without linbreak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "167ad479-6477-45eb-93a6-da078e16c816",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_hard_1 = ''\n",
    "\n",
    "for ei in hard_idx_1:\n",
    "    idx = train_step_dict[18][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_hard_1 += 'Question: ' + q + '\\n'\n",
    "    prompt_hard_1 += 'Options: ' + c + '\\n'\n",
    "    prompt_hard_1 += \"Let's think step by step\\n\"\n",
    "    prompt_hard_1 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "b5329f22-0f42-4b63-9d13-95b9dea22822",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a tank holds x gallons of a saltwater solution that is 20 % salt by volume . one fourth of the water is evaporated , leaving all of the salt . when 6 gallons of water and 12 gallons of salt are added , the resulting mixture is 33 1 / 3 % salt by volume . what is the value of x ?\n",
      "Options: a ) 37.5 , b ) 75 , c ) 100 , d ) 150 , e ) 90\n",
      "Let's think step by step\n",
      "nope , 150 . i can only get it by following pr ' s backsolving explanation . i hate that . original mixture has 20 % salt and 80 % water . total = x out of which salt = 0.2 x and water = 0.8 x now , 1 / 4 water evaporates and all salt remains . so what remains is 0.2 x salt and 0.6 x water . now 12 gallons salt is added and 6 gallons of water is added . so salt now becomes - > ( 0.2 x + 12 ) and water - - > ( 0.6 x + 6 ) amount of salt is 33.33 % of total . so amount of water is 66.66 % . so salt is half of the volume of water . so ( 0.2 x + 12 ) = ( 0.6 x + 6 ) / 2 = > 0.4 x + 24 = 0.6 x + 6 - > 0.2 x = 18 solving , x = 90 answer : e 90\n",
      "\n",
      "Question: what is the area inscribed by the lines y = 10 , x = 1 , y = 3 x + 1 on an xy - coordinate plane ?\n",
      "Options: a ) a ) 1 , b ) b ) 2 , c ) c ) 3 , d ) d ) 6 , e ) e ) 12\n",
      "Let's think step by step\n",
      "first , let ' s graph the lines y = 10 and x = 1 at this point , we need to find the points where the line y = 3 x + 1 intersects the other two lines . for the vertical line , we know that x = 1 , so we ' ll plug x = 1 into the equation y = 3 x + 1 to get y = 3 ( 1 ) + 1 = 4 perfect , when x = 1 , y = 5 , so one point of intersection is ( 1,4 ) for the horizontal line , we know that y = 10 , so we ' ll plug y = 10 into the equation y = 3 x + 1 to get 10 = 3 x + 1 . solve to get : x = 3 so , when y = 10 , x = 3 , so one point of intersection is ( 3,10 ) now add these points to our graph and sketch the line y = 3 x + 1 at this point , we can see that we have the following triangle . the base has length 2 and the height is 6 area = ( 1 / 2 ) ( base ) ( height ) = ( 1 / 2 ) ( 2 ) ( 6 ) = 6 answer : d 6 \n",
      "\n",
      "Question: machine z takes 2 more hours than machine b to make 20 widgets . if working together , the machines can make 25 widgets in 3 hours , how long will it take machine z to make 40 widgets ?\n",
      "Options: a ) 5 , b ) 6 , c ) 8 , d ) 10 , e ) 12\n",
      "Let's think step by step\n",
      "i approached this one by plugging in numbers . . . started with c . if 40 are made in 8 hours , then 20 are made in 4 hours . so time of z is 4 , and time of b is 2 . rate together : 20 / 4 + 20 / 2 = 5 + 10 = 15 . so in 1 hour , together make 15 widgets . in 3 hours = 45 . way too much . we can eliminate right away c , b , and a - because b and a reduces the time - the total # of widgets made will be even higher . now between d and e - > try only one . . if it does n ' t work , then the other one is the answer . i picked e : 12 h to make 40 widgets , and 6 hours to make 20 . this is the time of z . time of b = 4 hours . 20 / 6 + 20 / 4 = 10 / 3 + 20 / 4 find lcm of 3 and 4 = 12 . multiply first by 4 , and second by 3 : 40 + 60 / 12 = 100 / 12 divide by 4 : 25 / 3 so this is the rate given . e is the correct answer 12\n",
      "\n",
      "Question: lucia ' s sells kale at x dollar per pound for the first 20 pounds and . 8 x for every subsequent pound . amby ' s price is x per pound for the first 14 pounds and . 9 x for subsequent pounds . what is the minimum number of pounds over 15 for which lucia ' s becomes an equal or better deal ?\n",
      "Options: a ) 24 , b ) 26 , c ) 25 , d ) 27 , e ) 28\n",
      "Let's think step by step\n",
      "for amy ' s deal to be better , the cost has to be less or equal to lucia ' s assuming ' n ' is the number of pounds of kale , the equation is 20 x + ( n - 20 ) ( 0.8 x ) < = 14 x + ( n - 14 ) ( 0.9 x ) resolve it : = = > 20 x + 0.8 nx - 16 x < = 14 x + 0.9 nx - 12.6 x = = > 2.6 x < = 0.1 nx = = > 26 x < = nx = = > x ( n - 26 ) > = 0 as x can not be 0 , = = > n - 26 > = 0 = = > n > = 26 so the minimum value is 26 ' b ' would be the correct answer 26 \n",
      "\n",
      "Question: the function f ( n ) is defined as the product of all the consecutive positive integers between 2 and n ^ 2 , inclusive , whereas the function g ( n ) is defined as the product of the squares of all the consecutive positive integers between 1 and n , inclusive . the exponent on 2 in the prime factorization of f ( 3 ) / g ( 3 ) is\n",
      "Options: a ) 1 , b ) 2 , c ) 3 , d ) 4 , e ) 5\n",
      "Let's think step by step\n",
      "f ( 3 ) / g ( 3 ) = product ( 1 to 3 ^ 2 ) / 1.2 ^ 2.3 ^ 2 = 1 . 2.3 . 4.5 . 6.7 . 8.9 / 1 . 4.9 = 1 . 2.3 . ( 2 ^ 2 ) . 5 . ( 2.3 ) . 7 . ( 2 ^ 3 ) . 9 / 1 . ( 2 ^ 2 ) . 9 = 1 . ( 2 ^ 7 ) . 3.5 . 7.9 / 1 . ( 2 ^ 2 ) . 9 loof for 2 ^ 7 / 2 ^ 2 = 2 ^ 5 - - - - exponent 3 answer : c 3 \n",
      "\n",
      "Question: if x = 2 ^ 8 and x ^ x = 2 ^ k , what is k ?\n",
      "Options: a ) 2 ^ 15 , b ) 2 ^ 14 , c ) 2 ^ 13 , d ) 2 ^ 12 , e ) 2 ^ 11\n",
      "Let's think step by step\n",
      "solution : we know that x = 2 ^ 8 which implies x ^ x = ( 2 ^ 8 ) ^ ( 2 ^ 8 ) = 2 ^ ( 8 * 2 ^ 8 ) [ because ( x ^ y ) ^ z = x ^ ( y * z ) ) ] so 2 ^ ( 2 ^ 3 * 2 ^ 8 ) = 2 ^ ( 2 ^ ( 3 + 8 ) ) [ because x ^ a * x ^ b = x ^ ( a + b ) ] therefore x ^ x = 2 ^ ( 2 ^ 11 ) given that x ^ x = 2 ^ k so 2 ^ ( 2 ^ 11 ) = 2 ^ k since the base is same the exponent will also be same therefore k = 2 ^ 11 answer : e 2 ^ 11\n",
      "\n",
      "Question: a 60 kg metal bar made of alloy of tin and silver lost 6 kg of its weight in the water . 10 kg of tin loses 1.375 kg in the water ; 5 kg of silver loses 0.375 kg . what is the ratio of tin to silver in the bar ?\n",
      "Options: a ) 1 / 4 , b ) 2 / 5 , c ) 1 / 2 , d ) 3 / 5 , e ) 2 / 3\n",
      "Let's think step by step\n",
      "you can simply use this formula to avoid confusion : w 1 / w 2 = ( a 2 - aavg ) / ( avg - a 1 ) here is how you will find the values of a 1 an a 2 . we have an overall loss ( average loss ) . the average loss is 6 kg when 60 kg alloy is immersed . this is a loss of ( 6 / 60 ) * 100 = 10 % . this is aavg the loss of tin is 1.375 kg for every 10 kg . this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water . this is a 1 . the loss of silver is . 375 kg for every 5 kg . this means it loses ( . 375 / 5 ) * 100 = 7.5 % of its weight in water . this is a 2 . weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss ) x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3 e 2 / 3\n",
      "\n",
      "Question: a man has $ 10,000 to invest . he invests $ 4000 at 5 % and $ 3500 at 4 % . in order to have a yearly income of $ 500 , he must invest the remainder at :\n",
      "Options: a ) 4.6 % , b ) 3.6 % , c ) 6.4 % , d ) 7.8 % , e ) 8.9 %\n",
      "Let's think step by step\n",
      "c 6.4 % income from $ 4000 at 5 % in one year = $ 4000 of 5 % . = $ 4000 × 5 / 100 . = $ 4000 × 0.05 . = $ 200 . income from $ 3500 at 4 % in one year = $ 3500 of 4 % . = $ 3500 × 4 / 100 . = $ 3500 × 0.04 . = $ 140 . total income from 4000 at 5 % and 3500 at 4 % = $ 200 + $ 140 = $ 340 . remaining income amount in order to have a yearly income of $ 500 = $ 500 - $ 340 . = $ 160 . total invested amount = $ 4000 + $ 3500 = $ 7500 . remaining invest amount = $ 10000 - $ 7500 = $ 2500 . we know that , interest = principal × rate × time interest = $ 160 , principal = $ 2500 , rate = r [ we need to find the value of r ] , time = 1 year . 160 = 2500 × r × 1 . 160 = 2500 r 160 / 2500 = 2500 r / 2500 [ divide both sides by 2500 ] 0.064 = r r = 0.064 change it to a percent by moving the decimal to the right two places r = 6.4 % therefore , he invested the remaining amount $ 2500 at 6.4 % in order to get $ 500 income every year . 6.4 % \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_hard_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "72de84e6-2295-41f2-a31e-298a5894c8ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [21:04<00:00,  6.32s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_hard_1.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = prompt_hard_1 + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "2b858649-2568-481d-a5ef-d276dbce1614",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 220 correct 50 ratio 0.2273\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_hard_1, ans_gold_hard_1 = parse_pred_ans('outputs/dev_outputs_coht_hard_1.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "008a8615-4bba-4200-8589-ef1c1c273b5e",
   "metadata": {},
   "source": [
    "### With linebreak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "32c5cf7e-5569-4b6a-b977-16ba880987d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_hard_1_lb = [\n",
    "\"Question: a tank holds x gallons of a saltwater solution that is 20 % salt by volume . one fourth of the water is evaporated , leaving all of the salt . when 6 gallons of water and 12 gallons of salt are added , the resulting mixture is 33 1 / 3 % salt by volume . what is the value of x?\\n\",\n",
    "\"Options: a ) 37.5 , b ) 75 , c ) 100 , d ) 150 , e ) 90\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"original mixture has 20 % salt and 80 % water.\\n\", \n",
    "\"total = x out of which salt = 0.2x and water = 0.8x\\n\",\n",
    "\"now, 1/4 water evaporates and all salt remains.\\n\",\n",
    "\"so what remains is 0.2x salt and 0.6x water.\\n\",\n",
    "\"now 12 gallons salt is added and 6 gallons of water is added.\\n\",\n",
    "\"so salt now becomes (0.2x + 12) and water becomes (0.6x + 6)\\n\",\n",
    "\"amount of salt is 33.33% of total . so amount of water is 66.66%.\\n\" \n",
    "\"so salt is half of the volume of water.\\n\"\n",
    "\"so ( 0.2 x + 12 ) = ( 0.6 x + 6 ) / 2\\n\",\n",
    "\"0.4 x + 24 = 0.6 x + 6\\n\"\n",
    "\"0.2x = 18, solving it gives us x = 90\\n\",\n",
    "\"The answer is e, 90\\n\\n\",\n",
    "\n",
    "\"Question: what is the area inscribed by the lines y = 10 , x = 1 , y = 3 x + 1 on an xy - coordinate plane ?\\n\",\n",
    "\"Options: a ) 1 , b ) 2 , c ) 3 , d ) 6 , e ) 12\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"let' s graph the lines y = 10 and x = 1 at this point\\n\",\n",
    "\"we need to find the points where the line y = 3 x + 1 intersects the other two lines.\\n\",\n",
    "\"for the vertical line, we know that x = 1\\n\",\n",
    "\"so we' ll plug x = 1 into the equation y = 3 x + 1 to get y = 3 ( 1 ) + 1 = 4\\n\",\n",
    "\"when x = 1, y = 5, so one point of intersection is (1, 4)\\n\", \n",
    "\"for the horizontal line , we know that y = 10\\n\",\n",
    "\"so we' ll plug y = 10 into the equation y = 3x + 1 to get 10 = 3x + 1.\\n\", \n",
    "\"solve to get: x = 3\\n\",\n",
    "\"so, when y = 10 , x = 3, so one point of intersection is ( 3,10 ) now add these points to our graph\\n\", \n",
    "\"and sketch the line y = 3 x + 1 at this point, we can see that we have the following triangle.\\n\", \n",
    "\"the base has length 2 and the height is 6 area = (1 / 2) ( base ) ( height ) = ( 1 / 2 ) ( 2 ) ( 6 ) = 6\\n\",\n",
    "\"The answer is d, 6\\n\\n\",\n",
    "\n",
    "\"Question: machine z takes 2 more hours than machine b to make 20 widgets . if working together , the machines can make 25 widgets in 3 hours , how long will it take machine z to make 40 widgets ?\\n\",\n",
    "\"Options: a ) 5 , b ) 6 , c ) 8 , d ) 10 , e ) 12\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"started with c.\\n\",\n",
    "\"if 40 are made in 8 hours , then 20 are made in 4 hours.\\n\",\n",
    "\"so time of z is 4 , and time of b is 2.\\n\",\n",
    "\"rate together : 20 / 4 + 20 / 2 = 5 + 10 = 15.\\n\",\n",
    "\"so in 1 hour , together make 15 widgets.\\n\",\n",
    "\"in 3 hours = 45. way too much.\\n\", \n",
    "\"we can eliminate right away c , b , and a\\n\",\n",
    "\"because b and a reduces the time - the total # of widgets made will be even higher.\\n\",\n",
    "\"now between d and e, try only one.\\n\"\n",
    "\"if it doesn't work, then the other one is the answer.\\n\",\n",
    "\"i picked e: 12 h to make 40 widgets , and 6 hours to make 20. this is the time of z.\\n\",\n",
    "\"time of b = 4 hours.\\n\", \n",
    "\"20 / 6 + 20 / 4 = 10 / 3 + 20 / 4 find lcm of 3 and 4 = 12\\n\",\n",
    "\"multiply first by 4 , and second by 3 : 40 + 60 / 12 = 100 / 12\\n\", \n",
    "\"divide by 4 : 25 / 3 so this is the rate given\\n\",\n",
    "\"The answer is e, 12\\n\\n\",\n",
    "\n",
    "\"Question: lucia ' s sells kale at x dollar per pound for the first 20 pounds and . 8 x for every subsequent pound . amby ' s price is x per pound for the first 14 pounds and . 9 x for subsequent pounds . what is the minimum number of pounds over 15 for which lucia ' s becomes an equal or better deal ?\\n\",\n",
    "\"Options: a ) 24 , b ) 26 , c ) 25 , d ) 27 , e ) 28\\n\",\n",
    "\"Let's think step by step\",\n",
    "\"for amy' s deal to be better , the cost has to be less or equal to lucia's\\n\", \n",
    "\"assuming n is the number of pounds of kale\\n\",\n",
    "\"the equation is 20 x + ( n - 20 ) ( 0.8 x ) < = 14 x + ( n - 14 ) ( 0.9 x )\\n\",\n",
    "\"resolve it: 20 x + 0.8 nx - 16 x <= 14 x + 0.9 nx - 12.6 x\\n\",\n",
    "\"2.6 x <= 0.1 nx\\n\",\n",
    "\"26 x <= nx\\n\", \n",
    "\"x ( n - 26 ) > = 0\\n\", \n",
    "\"as x can not be 0, n - 26 >= 0\\n\",\n",
    "\"n >= 26, so the minimum value is 26\\n\",\n",
    "\"The answer is b, 26\\n\\n\",\n",
    "\n",
    "\"Question: the function f ( n ) is defined as the product of all the consecutive positive integers between 2 and n ^ 2 , inclusive , whereas the function g ( n ) is defined as the product of the squares of all the consecutive positive integers between 1 and n , inclusive . the exponent on 2 in the prime factorization of f ( 3 ) / g ( 3 ) is\\n\",\n",
    "\"Options: a ) 1 , b ) 2 , c ) 3 , d ) 4 , e ) 5\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"f(3) / g(3) = product (1 to 3 ^ 2) / 1 * 2^2 * 3^2\\n\",\n",
    "\"= 1 * 2 * 3 * 4 * 5 * 6 * 7 * 8 * 9 / 1 * 4 * 9\\n\", \n",
    "\"= 1 * 2 * 3 * ( 2 ^ 2 ) * 5 * ( 2 * 3 ) * 7 * ( 2 ^ 3 ) * 9 / 1 * ( 2 ^ 2 ) . 9\\n\",\n",
    "\"= 1 . ( 2 ^ 7 ) * 3 * 5 * 7 * 9 / 1 * ( 2 ^ 2 ) * 9\\n\", \n",
    "\"loof for 2 ^ 7 / 2 ^ 2 = 2 ^ 5, exponent 3\\n\",\n",
    "\"The answer is c, 3\\n\\n\"\n",
    "\n",
    "\"Question: if x = 2 ^ 8 and x ^ x = 2 ^ k , what is k ?\\n\",\n",
    "\"Options: a ) 2 ^ 15 , b ) 2 ^ 14 , c ) 2 ^ 13 , d ) 2 ^ 12 , e ) 2 ^ 11\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"we know that x = 2 ^ 8\\n\",\n",
    "\"which implies x ^ x = ( 2 ^ 8 ) ^ ( 2 ^ 8 ) = 2 ^ ( 8 * 2 ^ 8 )\\n\",\n",
    "\"because ( x ^ y ) ^ z = x ^ ( y * z ) )\\n\",\n",
    "\"so 2 ^ ( 2 ^ 3 * 2 ^ 8 ) = 2 ^ ( 2 ^ ( 3 + 8 ) )\\n\",\n",
    "\"because x ^ a * x ^ b = x ^ ( a + b )\\n\",\n",
    "\"therefore x ^ x = 2 ^ ( 2 ^ 11 )\\n\",\n",
    "\"given that x ^ x = 2 ^ k\\n\", \n",
    "\"so 2 ^ ( 2 ^ 11 ) = 2 ^ k\\n\",\n",
    "\"since the base is same the exponent will also be same\\n\",\n",
    "\"therefore k = 2 ^ 11\\n\",\n",
    "\"The answer is e, 2 ^ 11\\n\\n\"\n",
    "\n",
    "\"Question: a 60 kg metal bar made of alloy of tin and silver lost 6 kg of its weight in the water . 10 kg of tin loses 1.375 kg in the water ; 5 kg of silver loses 0.375 kg . what is the ratio of tin to silver in the bar ?\\n\"\n",
    "\"Options: a ) 1 / 4 , b ) 2 / 5 , c ) 1 / 2 , d ) 3 / 5 , e ) 2 / 3\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"you can simply use this formula to avoid confusion: w1 / w2 = ( a2 - aavg ) / ( avg - a1 )\\n\",\n",
    "\"here is how you will find the values of a1 an a2.\\n\", \n",
    "\"we have an overall loss ( average loss ).\\n\",\n",
    "\"the average loss is 6 kg when 60 kg alloy is immersed.\\n\", \n",
    "\"this is a loss of ( 6 / 60 ) * 100 = 10 %.\\n\",\n",
    "\"this is aavg the loss of tin is 1.375 kg for every 10 kg.\\n\", \n",
    "\"this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water.\\n\",\n",
    "\"this is a1.\\n\",\n",
    "\"the loss of silver is .375 kg for every 5 kg.\\n\", \n",
    "\"this means it loses ( .375 / 5 ) * 100 = 7.5 % of its weight in water.\\n\",\n",
    "\"this is a2.\\n\",\n",
    "\"weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss )\\n\",\n",
    "\"x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3\\n\",\n",
    "\"The answer is e, 2 / 3\\n\\n\",\n",
    "\n",
    "\"Question: a man has $ 10,000 to invest . he invests $ 4000 at 5 % and $ 3500 at 4 % . in order to have a yearly income of $ 500 , he must invest the remainder at :\\n\",\n",
    "\"Options: a ) 4.6 % , b ) 3.6 % , c ) 6.4 % , d ) 7.8 % , e ) 8.9 %\\n\",\n",
    "\"Let's think step by step\\n\"\n",
    "\"income from $4000 at 5% in one year = $4000 of 5% = $4000 × 5 / 100 = $4000 × 0.05 = $200.\\n\",\n",
    "\"income from $ 3500 at 4 % in one year = $3500 of 4% = $3500 × 4 / 100 = $3500 × 0.04 = $140.\\n\",\n",
    "\"total income from 4000 at 5% and 3500 at 4% = $200 + $140 = $340.\\n\",\n",
    "\"remaining income amount in order to have a yearly income of $500 is $500 - $340 = $160.\\n\",\n",
    "\"total invested amount = $4000 + $3500 = $7500.\\n\",\n",
    "\"remaining invest amount = $10000 - $7500 = $2500.\\n\",\n",
    "\"we know that, interest = principal × rate × time interest = $160\\n\",\n",
    "\"principal = $2500 , rate = r\\n\",\n",
    "\"we need to find the value of r\\n\",\n",
    "\"time = 1 year . 160 = 2500 × r × 1\\n\", \n",
    "\"160 = 2500r\\n\", \n",
    "\"160 / 2500 = 2500 r / 2500 (divide both sides by 2500)\\n\",\n",
    "\"r = 0.064\\n\",\n",
    "\"change it to a percent by moving the decimal to the right two places r = 6.4%\\n\", \n",
    "\"therefore , he invested the remaining amount $2500 at 6.4% in order to get $500 income every year.\\n\", \n",
    "\"The answer is c, 6.4%\\n\\n\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "2a33e8f1-1967-4b4d-863b-2a00b4e628b9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a tank holds x gallons of a saltwater solution that is 20 % salt by volume . one fourth of the water is evaporated , leaving all of the salt . when 6 gallons of water and 12 gallons of salt are added , the resulting mixture is 33 1 / 3 % salt by volume . what is the value of x?\n",
      "Options: a ) 37.5 , b ) 75 , c ) 100 , d ) 150 , e ) 90\n",
      "Let's think step by step\n",
      "original mixture has 20 % salt and 80 % water.\n",
      "total = x out of which salt = 0.2x and water = 0.8x\n",
      "now, 1/4 water evaporates and all salt remains.\n",
      "so what remains is 0.2x salt and 0.6x water.\n",
      "now 12 gallons salt is added and 6 gallons of water is added.\n",
      "so salt now becomes (0.2x + 12) and water becomes (0.6x + 6)\n",
      "amount of salt is 33.33% of total . so amount of water is 66.66%.\n",
      "so salt is half of the volume of water.\n",
      "so ( 0.2 x + 12 ) = ( 0.6 x + 6 ) / 2\n",
      "0.4 x + 24 = 0.6 x + 6\n",
      "0.2x = 18, solving it gives us x = 90\n",
      "The answer is e, 90\n",
      "\n",
      "Question: what is the area inscribed by the lines y = 10 , x = 1 , y = 3 x + 1 on an xy - coordinate plane ?\n",
      "Options: a ) 1 , b ) 2 , c ) 3 , d ) 6 , e ) 12\n",
      "Let's think step by step\n",
      "let' s graph the lines y = 10 and x = 1 at this point\n",
      "we need to find the points where the line y = 3 x + 1 intersects the other two lines.\n",
      "for the vertical line, we know that x = 1\n",
      "so we' ll plug x = 1 into the equation y = 3 x + 1 to get y = 3 ( 1 ) + 1 = 4\n",
      "when x = 1, y = 5, so one point of intersection is (1, 4)\n",
      "for the horizontal line , we know that y = 10\n",
      "so we' ll plug y = 10 into the equation y = 3x + 1 to get 10 = 3x + 1.\n",
      "solve to get: x = 3\n",
      "so, when y = 10 , x = 3, so one point of intersection is ( 3,10 ) now add these points to our graph\n",
      "and sketch the line y = 3 x + 1 at this point, we can see that we have the following triangle.\n",
      "the base has length 2 and the height is 6 area = (1 / 2) ( base ) ( height ) = ( 1 / 2 ) ( 2 ) ( 6 ) = 6\n",
      "The answer is d, 6\n",
      "\n",
      "Question: machine z takes 2 more hours than machine b to make 20 widgets . if working together , the machines can make 25 widgets in 3 hours , how long will it take machine z to make 40 widgets ?\n",
      "Options: a ) 5 , b ) 6 , c ) 8 , d ) 10 , e ) 12\n",
      "Let's think step by step\n",
      "started with c.\n",
      "if 40 are made in 8 hours , then 20 are made in 4 hours.\n",
      "so time of z is 4 , and time of b is 2.\n",
      "rate together : 20 / 4 + 20 / 2 = 5 + 10 = 15.\n",
      "so in 1 hour , together make 15 widgets.\n",
      "in 3 hours = 45. way too much.\n",
      "we can eliminate right away c , b , and a\n",
      "because b and a reduces the time - the total # of widgets made will be even higher.\n",
      "now between d and e, try only one.\n",
      "if it doesn't work, then the other one is the answer.\n",
      "i picked e: 12 h to make 40 widgets , and 6 hours to make 20. this is the time of z.\n",
      "time of b = 4 hours.\n",
      "20 / 6 + 20 / 4 = 10 / 3 + 20 / 4 find lcm of 3 and 4 = 12\n",
      "multiply first by 4 , and second by 3 : 40 + 60 / 12 = 100 / 12\n",
      "divide by 4 : 25 / 3 so this is the rate given\n",
      "The answer is e, 12\n",
      "\n",
      "Question: lucia ' s sells kale at x dollar per pound for the first 20 pounds and . 8 x for every subsequent pound . amby ' s price is x per pound for the first 14 pounds and . 9 x for subsequent pounds . what is the minimum number of pounds over 15 for which lucia ' s becomes an equal or better deal ?\n",
      "Options: a ) 24 , b ) 26 , c ) 25 , d ) 27 , e ) 28\n",
      "Let's think step by stepfor amy' s deal to be better , the cost has to be less or equal to lucia's\n",
      "assuming n is the number of pounds of kale\n",
      "the equation is 20 x + ( n - 20 ) ( 0.8 x ) < = 14 x + ( n - 14 ) ( 0.9 x )\n",
      "resolve it: 20 x + 0.8 nx - 16 x <= 14 x + 0.9 nx - 12.6 x\n",
      "2.6 x <= 0.1 nx\n",
      "26 x <= nx\n",
      "x ( n - 26 ) > = 0\n",
      "as x can not be 0, n - 26 >= 0\n",
      "n >= 26, so the minimum value is 26\n",
      "The answer is b, 26\n",
      "\n",
      "Question: the function f ( n ) is defined as the product of all the consecutive positive integers between 2 and n ^ 2 , inclusive , whereas the function g ( n ) is defined as the product of the squares of all the consecutive positive integers between 1 and n , inclusive . the exponent on 2 in the prime factorization of f ( 3 ) / g ( 3 ) is\n",
      "Options: a ) 1 , b ) 2 , c ) 3 , d ) 4 , e ) 5\n",
      "Let's think step by step\n",
      "f(3) / g(3) = product (1 to 3 ^ 2) / 1 * 2^2 * 3^2\n",
      "= 1 * 2 * 3 * 4 * 5 * 6 * 7 * 8 * 9 / 1 * 4 * 9\n",
      "= 1 * 2 * 3 * ( 2 ^ 2 ) * 5 * ( 2 * 3 ) * 7 * ( 2 ^ 3 ) * 9 / 1 * ( 2 ^ 2 ) . 9\n",
      "= 1 . ( 2 ^ 7 ) * 3 * 5 * 7 * 9 / 1 * ( 2 ^ 2 ) * 9\n",
      "loof for 2 ^ 7 / 2 ^ 2 = 2 ^ 5, exponent 3\n",
      "The answer is c, 3\n",
      "\n",
      "Question: if x = 2 ^ 8 and x ^ x = 2 ^ k , what is k ?\n",
      "Options: a ) 2 ^ 15 , b ) 2 ^ 14 , c ) 2 ^ 13 , d ) 2 ^ 12 , e ) 2 ^ 11\n",
      "Let's think step by step\n",
      "we know that x = 2 ^ 8\n",
      "which implies x ^ x = ( 2 ^ 8 ) ^ ( 2 ^ 8 ) = 2 ^ ( 8 * 2 ^ 8 )\n",
      "because ( x ^ y ) ^ z = x ^ ( y * z ) )\n",
      "so 2 ^ ( 2 ^ 3 * 2 ^ 8 ) = 2 ^ ( 2 ^ ( 3 + 8 ) )\n",
      "because x ^ a * x ^ b = x ^ ( a + b )\n",
      "therefore x ^ x = 2 ^ ( 2 ^ 11 )\n",
      "given that x ^ x = 2 ^ k\n",
      "so 2 ^ ( 2 ^ 11 ) = 2 ^ k\n",
      "since the base is same the exponent will also be same\n",
      "therefore k = 2 ^ 11\n",
      "The answer is e, 2 ^ 11\n",
      "\n",
      "Question: a 60 kg metal bar made of alloy of tin and silver lost 6 kg of its weight in the water . 10 kg of tin loses 1.375 kg in the water ; 5 kg of silver loses 0.375 kg . what is the ratio of tin to silver in the bar ?\n",
      "Options: a ) 1 / 4 , b ) 2 / 5 , c ) 1 / 2 , d ) 3 / 5 , e ) 2 / 3\n",
      "Let's think step by step\n",
      "you can simply use this formula to avoid confusion: w1 / w2 = ( a2 - aavg ) / ( avg - a1 )\n",
      "here is how you will find the values of a1 an a2.\n",
      "we have an overall loss ( average loss ).\n",
      "the average loss is 6 kg when 60 kg alloy is immersed.\n",
      "this is a loss of ( 6 / 60 ) * 100 = 10 %.\n",
      "this is aavg the loss of tin is 1.375 kg for every 10 kg.\n",
      "this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water.\n",
      "this is a1.\n",
      "the loss of silver is .375 kg for every 5 kg.\n",
      "this means it loses ( .375 / 5 ) * 100 = 7.5 % of its weight in water.\n",
      "this is a2.\n",
      "weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss )\n",
      "x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3\n",
      "The answer is e, 2 / 3\n",
      "\n",
      "Question: a man has $ 10,000 to invest . he invests $ 4000 at 5 % and $ 3500 at 4 % . in order to have a yearly income of $ 500 , he must invest the remainder at :\n",
      "Options: a ) 4.6 % , b ) 3.6 % , c ) 6.4 % , d ) 7.8 % , e ) 8.9 %\n",
      "Let's think step by step\n",
      "income from $4000 at 5% in one year = $4000 of 5% = $4000 × 5 / 100 = $4000 × 0.05 = $200.\n",
      "income from $ 3500 at 4 % in one year = $3500 of 4% = $3500 × 4 / 100 = $3500 × 0.04 = $140.\n",
      "total income from 4000 at 5% and 3500 at 4% = $200 + $140 = $340.\n",
      "remaining income amount in order to have a yearly income of $500 is $500 - $340 = $160.\n",
      "total invested amount = $4000 + $3500 = $7500.\n",
      "remaining invest amount = $10000 - $7500 = $2500.\n",
      "we know that, interest = principal × rate × time interest = $160\n",
      "principal = $2500 , rate = r\n",
      "we need to find the value of r\n",
      "time = 1 year . 160 = 2500 × r × 1\n",
      "160 = 2500r\n",
      "160 / 2500 = 2500 r / 2500 (divide both sides by 2500)\n",
      "r = 0.064\n",
      "change it to a percent by moving the decimal to the right two places r = 6.4%\n",
      "therefore , he invested the remaining amount $2500 at 6.4% in order to get $500 income every year.\n",
      "The answer is c, 6.4%\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join(prompt_hard_1_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "a38dfa97-9bb2-4eaa-8dce-00e31426066e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [20:04<00:00,  6.02s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_hard_1_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_hard_1_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "e15880e9-48b3-434e-b9c8-82476b1dc2e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 63 ratio 0.3150\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_hard_1, ans_gold_hard_1 = parse_pred_ans('outputs/dev_outputs_coht_hard_1_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "237ed642-976a-4c98-9b63-3a6c14c441b7",
   "metadata": {},
   "source": [
    "## Hard 2 reduce the number of equations, Acc 31.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "1773c524-2166-4728-a814-1b1aeaa96860",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_hard_2 = ''\n",
    "\n",
    "for ei in hard_idx_2:\n",
    "    idx = train_step_dict[18][ei]\n",
    "    q = train_data[idx]['Problem']\n",
    "    c = train_data[idx]['options']\n",
    "    cr = train_data[idx]['correct']\n",
    "    r = train_data[idx]['Rationale']\n",
    "    \n",
    "    ans_dict = {}\n",
    "    for ac in c.split(', '):\n",
    "        idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        if('a' in idx_): ans_dict['a'] = num_\n",
    "        if('b' in idx_): ans_dict['b'] = num_\n",
    "        if('c' in idx_): ans_dict['c'] = num_\n",
    "        if('d' in idx_): ans_dict['d'] = num_\n",
    "        if('e' in idx_): ans_dict['e'] = num_\n",
    "        \n",
    "    if(r[0] == '\"'): r = r[1:]\n",
    "    if(r[-1] == '\"'): r = r[:-1]\n",
    "    \n",
    "    r += ' ' + ans_dict[cr]\n",
    "    \n",
    "    prompt_hard_2 += 'Question: ' + q + '\\n'\n",
    "    prompt_hard_2 += 'Options: ' + c + '\\n'\n",
    "    prompt_hard_2 += \"Let's think step by step\\n\"\n",
    "    prompt_hard_2 += r + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "54aa3610-9f6d-4052-857e-09a6be5430fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: the difference in compound interest earned on a deposit ( compounded annually ) in year 1 and year 2 is $ 40 . had the interest rate been 3 times its present value , the difference c would have been how much ?\n",
      "Options: a ) 40 / 3 , b ) 40 , c ) 120 , d ) 360 , e ) 420\n",
      "Let's think step by step\n",
      "case 1 : deposit = $ x ; rate of increase = r . interest yearned in 1 year = xr . deposit in 1 year = x + xr . interest yearned in 2 year = ( x + xr ) r . the difference c = ( x + xr ) r - xr = xr ^ 2 = 40 . case 2 : deposit = $ x ; rate of increase = 3 r . interest yearned in 1 year = x ( 3 r ) . deposit in 1 year = x + 3 xr . interest yearned in 2 year = ( x + 3 xr ) 3 r . the difference = ( x + 3 xr ) 3 r - 3 xr = 9 xr ^ 2 . since from case 1 we know that xr ^ 2 = 40 , then 9 xr ^ 2 = 9 * 40 = 360 . answer : d . 360 \n",
      "\n",
      "Question: s ( n ) is a n - digit number formed by attaching the first n perfect squares , in order , into one integer . for example , s ( 1 ) = 1 , s ( 2 ) = 14 , s ( 3 ) = 149 , s ( 4 ) = 14916 , s ( 5 ) = 1491625 , etc . how many digits r are in s ( 99 ) ?\n",
      "Options: a ) r = 350 , b ) r = 353 , c ) r = 354 , d ) r = 356 , e ) 357\n",
      "Let's think step by step\n",
      "focus on the points where the number of digits in squares change : 1 , 2 , 3 - single digit squares . first 2 digit number is 10 . 4 , 5 , . . . 9 - two digit squares . to get 9 , the last number with two digit square , think that first 3 digit number is 100 which is 10 ^ 2 . so 9 ^ 2 must be the last 2 digit square . 10 , 11 , 12 , . . . 31 - three digit squares . to get 31 , think of 1000 - the first 4 digit number . it is not a perfect square but 900 is 30 ^ 2 . 32 ^ 2 = 2 ^ 10 = 1024 , the first 4 digit square . 32 - 99 - four digit squares . to get 99 , think of 10,000 - the first 5 digit number which is 100 ^ 2 . so number of digits in s ( 99 ) = 3 * 1 + 6 * 2 + 22 * 3 + 68 * 4 = 3 + 12 + 66 + 272 = r = 353 . b r = 353 \n",
      "\n",
      "Question: a man invests rs . 8,000 at the rate of 5 % per annum . how much more should he invest at the rate of 8 % , so that he can earn a total of 6 % per annum ?\n",
      "Options: a ) rs . 1200 , b ) rs . 1300 , c ) rs . 1500 , d ) rs . 4000 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : interest on rs . 8000 at 5 % per annum = ( 8000 × 5 × 1 ) / 100 = rs . 400 let his additional investment at 8 % = x interest on rs . x at 8 % per annum = ( x × 8 × 1 ) / 100 = 2 x / 25 . to earn 6 % per annum for the total , interest = ( 8000 + x ) × 6 × 1 / 100 . = > 400 + 2 x / 25 = ( 8000 + x ) × 6 × 1 / 100 . = > 40000 + 8 x = ( 8000 + x ) × 6 . = > 40000 + 8 x = 48000 + 6 x . = > 2 x = 8000 . = > x = 4000 . answer : d rs . 4000 \n",
      "\n",
      "Question: mr . john used to purchase certain number of mangoes for $ 360 since the price of mangoes is reduced by 10 % he got 12 more mangoes today . find the original price of 125 mangoes .\n",
      "Options: a ) 360 , b ) 380 , c ) 400 , d ) 416 , e ) 412\n",
      "Let's think step by step\n",
      "mr . john used to purchase certain number of mangoes for $ 360 since the price of mangoes is reduced by 10 % he got 12 more mangoes today . find the original price of 120 mangoes . method 1 : let price per mango = x . let number of mangoes be n . then , nx = 360 . now price = 0.9 x ; number of mangoes = n + 12 . total amount = 0.9 x * ( n + 12 ) = 360 . nx = 0.9 nx + 10.8 x = > 0.1 nx = 10.8 x = > n = 108 = > x = 360 / 108 = 3.33 original price of 125 mangoes = 125 * 3.33 = 416 . answer d 416 \n",
      "\n",
      "Question: how many trailing zeroes does 53 ! + 54 ! have ?\n",
      "Options: a ) 12 , b ) 13 , c ) 14 , d ) 15 , e ) 16\n",
      "Let's think step by step\n",
      "53 ! + 54 ! = 53 ! + 54 * 53 ! = 53 ! ( 1 + 54 ) = 53 ! * 55 number of trailing 0 s in 53 ! = number of 5 s in the expansion of 53 ! = 10 + 2 = 12 there is 1 more 5 in 55 . hence , total number of trailing 0 s = 12 + 1 = 13 answer ( b ) . in most cases , when we are adding multiple terms , all of which have trailing 0 s , the sum will have as many trailing 0 s as that in the lowest term . example : 20 + 2300 + 34210000 = 34212320 - > one 0 because lowest term 20 has one 0 . this case has an exception because of an additional 5 in 55 . answer is b 13 \n",
      "\n",
      "Question: machine t takes 2 more hours than machine b to make 20 widgets . if working together , the machines can make 25 widgets in 3 hours , how long will it take machine t to make 40 widgets ?\n",
      "Options: a ) 5 , b ) 6 , c ) 8 , d ) 10 , e ) 12\n",
      "Let's think step by step\n",
      "i approached this one by plugging in numbers . . . started with c . if 40 are made in 8 hours , then 20 are made in 4 hours . so time of t is 4 , and time of b is 2 . rate together : 20 / 4 + 20 / 2 = 5 + 10 = 15 . so in 1 hour , together make 15 widgets . in 3 hours = 45 . way too much . we can eliminate right away c , b , and a - because b and t reduces the time - the total # of widgets made will be even higher . now between d and e - > try only one . . if it does n ' t work , then the other one is the answer . i picked e : 12 h to make 40 widgets , and 6 hours to make 20 . this is the time of t . time of b = 4 hours . 20 / 6 + 20 / 4 = 10 / 3 + 20 / 4 find lcm of 3 and 4 = 12 . multiply first by 4 , and second by 3 : 40 + 60 / 12 = 100 / 12 divide by 4 : 25 / 3 so this is the rate given . e is the correct answer 12\n",
      "\n",
      "Question: tim and é lan are 90 miles away from one another . they are starting to move towards each other simultaneously , tim at a speed of 10 mph and é lan at a speed of 5 mph . if every hour they double their speeds , what is the distance that tim will pass until he meets é lan ?\n",
      "Options: a ) 30 miles . , b ) 35 miles . , c ) 45 miles . , d ) 60 miles . , e ) 65 miles .\n",
      "Let's think step by step\n",
      "in 1 st hour : relative speed of the two will be 10 + 5 = 15 miles / hour . as they move towards each other , the distance between them reduces by 15 miles . distance left between them : 90 - 15 = 75 miles . in 2 nd hour : speed of tim : 20 mph , that of elan : 10 mph . relative speed of the two will be 20 + 10 = 30 miles / hour . as they move towards each other , the distance between them reduces by 30 miles . distance left between them : 75 - 30 = 45 miles . in 3 rd hour : speed of tim : 40 mph , that of elan : 20 mph . relative speed of the two will be 40 + 20 = 60 miles / hour . now in order to cover the remaining distance of 45 miles . they will take : ( 45 / 60 ) hrs . = ( 3 / 4 ) hour . distance travelled by tim ? in first hour : 1 hour * ( 10 miles per hour ) = 10 miles in second hour : 1 hour * ( 20 miles per hour ) = 20 miles in third hour : ( 3 / 4 ) hour * ( 40 miles per hour ) = 30 miles total distance travelled : 10 + 20 + 30 = 60 miles . d is the answer 60 miles . \n",
      "\n",
      "Question: what is the sum of all 3 digit numbers that leave a remainder of ' 2 ' when divided by 3 ?\n",
      "Options: a ) 897 , b ) 164,850 , c ) 164,749 , d ) 149,700 , e ) 156,720\n",
      "Let's think step by step\n",
      "explanatory answer step 1 : identify the series the smallest 3 digit number that will leave a remainder of 2 when divided by 3 is 101 . the next couple of numbers that will leave a remainder of 2 when divided by 3 are 104 and 107 . the largest 3 digit number that will leave a remainder of 2 when divided by 3 is 998 . it is evident that any number in the sequence will be a 3 digit positive integer of the form ( 3 n + 2 ) . so , the given numbers are in an arithmetic sequence with the first term being 101 and the last term being 998 and the common difference being 3 . step 2 : compute the sum sum of an arithmetic progression ( ap ) = [ first term + last term / 2 ] n we know the first term : 101 we know the last term : 998 . the only unknown is the number of terms , n . in an a . p . , the nth term an = a 1 + ( n - 1 ) * d in this case , therefore , 998 = 101 + ( n - 1 ) * 3 or 897 = ( n - 1 ) * 3 ( n - 1 ) = 299 or n = 300 . sum of the ap will therefore , be [ 101 + 998 / 2 ] ∗ 300 = 164,850 choice b is the correct answer . 164,850 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_hard_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "d6e31217-10cd-4cdd-a418-b709adca5bb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_hard_2_lb = [\n",
    "\"Question: the difference in compound interest earned on a deposit ( compounded annually ) in year 1 and year 2 is $ 40 . had the interest rate been 3 times its present value , the difference c would have been how much ?\\n\",\n",
    "\"Options: a ) 40 / 3 , b ) 40 , c ) 120 , d ) 360 , e ) 420\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"case 1 : deposit = $x\\n\",\n",
    "\"rate of increase = r\\n\",\n",
    "\"interest yearned in 1 year = xr\\n\"\n",
    "\"deposit in 1 year = x + xr\\n\",\n",
    "\"interest yearned in 2 year = ( x + xr ) r\\n\",\n",
    "\"the difference c = ( x + xr ) r - xr = xr ^ 2 = 40\\n\",\n",
    "\"case 2 : deposit = $x\\n,\",\n",
    "\"rate of increase = 3r\\n\",\n",
    "\"interest yearned in 1 year = x ( 3 r )\\n\",\n",
    "\"deposit in 1 year = x + 3 xr\\n\",\n",
    "\"interest yearned in 2 year = ( x + 3 xr ) 3 r\\n\",\n",
    "\"the difference = ( x + 3 xr ) 3 r - 3 xr = 9 xr ^ 2\\n\",\n",
    "\"since from case 1 we know that xr ^ 2 = 40\\n\",\n",
    "\"then 9 xr ^ 2 = 9 * 40 = 360.\\n\"\n",
    "\"The answer is d, 360\\n\\n\"\n",
    "\n",
    "\"Question: a man invests rs . 8,000 at the rate of 5 % per annum . how much more should he invest at the rate of 8 % , so that he can earn a total of 6 % per annum ?\\n\",\n",
    "\"Options: a ) rs . 1200 , b ) rs . 1300 , c ) rs . 1500 , d ) rs . 4000 , e ) none of these\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"interest on rs . 8000 at 5 % per annum = ( 8000 × 5 × 1 ) / 100 = rs . 400\\n\",\n",
    "\"let his additional investment at 8 % = x\\n\",\n",
    "\"interest on rs . x at 8 % per annum = ( x × 8 × 1 ) / 100 = 2 x / 25\\n\", \n",
    "\"to earn 6 % per annum for the total, interest = ( 8000 + x ) × 6 × 1 / 100.\\n\",\n",
    "\"400 + 2 x / 25 = ( 8000 + x ) × 6 × 1 / 100.\\n\",\n",
    "\"40000 + 8 x = ( 8000 + x ) × 6\\n\",\n",
    "\"40000 + 8 x = 48000 + 6 x\\n\",\n",
    "\"2 x = 8000. x = 4000.\\n\" \n",
    "\"The answer is d, rs . 4000\\n\\n\"\n",
    "\n",
    "\"Question: mr . john used to purchase certain number of mangoes for $ 360 since the price of mangoes is reduced by 10 % he got 12 more mangoes today . find the original price of 125 mangoes.\\n\"\n",
    "\"Options: a ) 360 , b ) 380 , c ) 400 , d ) 416 , e ) 412\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"mr. john used to purchase certain number of mangoes for $ 360\\n\",\n",
    "\"since the price of mangoes is reduced by 10 %\\n\",\n",
    "\"he got 12 more mangoes today.\\n\",\n",
    "\"find the original price of 120 mangoes.\\n\",\n",
    "\"method 1 : let price per mango = x.\\n\",\n",
    "\"let number of mangoes be n.\\n\",\n",
    "\"then , nx = 360.\\n\",\n",
    "\"now price = 0.9 x\\n\",\n",
    "\"number of mangoes = n + 12\\n\", \n",
    "\"total amount = 0.9 x * ( n + 12 ) = 360\\n\",\n",
    "\"nx = 0.9 nx + 10.8 x => 0.1 nx = 10.8 x => n = 108\\n\",\n",
    "\"x = 360 / 108 = 3.33\\n\",\n",
    "\"original price of 125 mangoes = 125 * 3.33 = 416.\\n\"\n",
    "\"The answer is d, 416\\n\\n\"\n",
    "\n",
    "# \"Question: machine t takes 2 more hours than machine b to make 20 widgets . if working together , the machines can make 25 widgets in 3 hours , how long will it take machine t to make 40 widgets ?\\n\",\n",
    "# \"Options: a ) 5 , b ) 6 , c ) 8 , d ) 10 , e ) 12\\n\",\n",
    "# \"Let's think step by step\\n\",\n",
    "# \"started with c.\\n\",\n",
    "# \"if 40 are made in 8 hours , then 20 are made in 4 hours.\\n\",\n",
    "# \"so time of t is 4 , and time of b is 2.\\n\",\n",
    "# \"rate together: 20 / 4 + 20 / 2 = 5 + 10 = 15.\\n\",\n",
    "# \"so in 1 hour , together make 15 widgets. in 3 hours = 45, way too much\\n\",\n",
    "# \"we can eliminate right away c , b , and a - because b and t reduces the time - the total # of widgets made will be even higher\\n\",\n",
    "# \"now between d and e - > try only one\\n\",\n",
    "# \"if it doesn' t work , then the other one is the answer.\\n\",\n",
    "# \"12 h to make 40 widgets , and 6 hours to make 20.\\n\",\n",
    "# \"this is the time of t. time of b = 4 hours.\\n\",\n",
    "# \"20 / 6 + 20 / 4 = 10 / 3 + 20 / 4 find lcm of 3 and 4 = 12.\\n\",\n",
    "# \"multiply first by 4 , and second by 3 : 40 + 60 / 12 = 100 / 12\\n\",\n",
    "# \"divide by 4 : 25 / 3 so this is the rate given .\\n\",\n",
    "# \"The answer is e, 12\\n\\n\"\n",
    "\n",
    "\"Question: tim and é lan are 90 miles away from one another . they are starting to move towards each other simultaneously , tim at a speed of 10 mph and é lan at a speed of 5 mph . if every hour they double their speeds , what is the distance that tim will pass until he meets é lan ?\\n\",\n",
    "\"Options: a ) 30 miles . , b ) 35 miles . , c ) 45 miles . , d ) 60 miles . , e ) 65 miles .\\n\",\n",
    "\"Let's think step by step\\n\"\n",
    "\"in 1 st hour: relative speed of the two will be 10 + 5 = 15 miles / hour\\n\",\n",
    "\"as they move towards each other, the distance between them reduces by 15 miles.\\n\",\n",
    "\"distance left between them: 90 - 15 = 75 miles.\\n\", \n",
    "\"in 2 d hour: speed of tim: 20 mph, that of elan : 10 mph.\\n\", \n",
    "\"relative speed of the two will be 20 + 10 = 30 miles / hour.\\n\",\n",
    "\"as they move towards each other , the distance between them reduces by 30 miles.\\n\",\n",
    "\"distance left between them : 75 - 30 = 45 miles.\\n\",\n",
    "\"in 3 rd hour: speed of tim: 40 mph , that of elan: 20 mph.\\n\",\n",
    "\"relative speed of the two will be 40 + 20 = 60 miles / hour.\\n\",\n",
    "\"now in order to cover the remaining distance of 45 miles . they will take : ( 45 / 60 ) hrs . = ( 3 / 4 ) hour.\\n\",\n",
    "\"distance travelled by tim in first hour : 1 hour * ( 10 miles per hour ) = 10 miles\\n\",\n",
    "\"in second hour : 1 hour * ( 20 miles per hour ) = 20 miles\\n\",\n",
    "\"in third hour : ( 3 / 4 ) hour * ( 40 miles per hour ) = 30 miles\\n\",\n",
    "\"total distance travelled : 10 + 20 + 30 = 60 miles.\\n\",\n",
    "\"The answer is d, 60 miles.\\n\\n\", \n",
    "\n",
    "\"Question: what is the sum of all 3 digit numbers that leave a remainder of ' 2 ' when divided by 3 ?\\n\"\n",
    "\"Options: a ) 897 , b ) 164,850 , c ) 164,749 , d ) 149,700 , e ) 156,720\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"step 1 : identify the series the smallest 3 digit number that will leave a remainder of 2 when divided by 3 is 101.\\n\",\n",
    "\"the next couple of numbers that will leave a remainder of 2 when divided by 3 are 104 and 107.\\n\", \n",
    "\"the largest 3 digit number that will leave a remainder of 2 when divided by 3 is 998.\\n\",\n",
    "\"it is evident that any number in the sequence will be a 3 digit positive integer of the form ( 3 n + 2 ).\\n\",\n",
    "\"so , the given numbers are in an arithmetic sequence with the first term being 101 and the last term being 998 and the common difference being 3.\\n\",\n",
    "\"step 2 : compute the sum sum of an arithmetic progression ( ap ) = [ first term + last term / 2 ] n \\n\",\n",
    "\"we know the first term : 101\\n\",\n",
    "\"we know the last term : 998\\n\",\n",
    "\"the only unknown is the number of terms , n .\\n\",\n",
    "\"in an a . p . , the nth term an = a 1 + ( n - 1 ) * d\\n\",\n",
    "\"in this case , therefore , 998 = 101 + ( n - 1 ) * 3 or 897 = ( n - 1 ) * 3 ( n - 1 ) = 299 or n = 300.\\n\",\n",
    "\"sum of the ap will therefore , be [ 101 + 998 / 2 ] ∗ 300 = 164,850\\n\",\n",
    "\"The answer is b, 164,850\\n\\n\"\n",
    "    \n",
    "\"Question: what is the area inscribed by the lines y = 10 , x = 1 , y = 3 x + 1 on an xy - coordinate plane ?\\n\",\n",
    "\"Options: a ) 1 , b ) 2 , c ) 3 , d ) 6 , e ) 12\\n\",\n",
    "\"Let's think step by step\\n\",\n",
    "\"let' s graph the lines y = 10 and x = 1 at this point\\n\",\n",
    "\"we need to find the points where the line y = 3 x + 1 intersects the other two lines.\\n\",\n",
    "\"for the vertical line, we know that x = 1\\n\",\n",
    "\"so we' ll plug x = 1 into the equation y = 3 x + 1 to get y = 3 ( 1 ) + 1 = 4\\n\",\n",
    "\"when x = 1, y = 5, so one point of intersection is (1, 4)\\n\", \n",
    "\"for the horizontal line , we know that y = 10\\n\",\n",
    "\"so we' ll plug y = 10 into the equation y = 3x + 1 to get 10 = 3x + 1.\\n\", \n",
    "\"solve to get: x = 3\\n\",\n",
    "\"so, when y = 10 , x = 3, so one point of intersection is ( 3,10 ) now add these points to our graph\\n\", \n",
    "\"and sketch the line y = 3 x + 1 at this point, we can see that we have the following triangle.\\n\", \n",
    "\"the base has length 2 and the height is 6 area = (1 / 2) ( base ) ( height ) = ( 1 / 2 ) ( 2 ) ( 6 ) = 6\\n\",\n",
    "\"The answer is d, 6\\n\\n\",\n",
    "    \n",
    "\"Question: a 60 kg metal bar made of alloy of tin and silver lost 6 kg of its weight in the water . 10 kg of tin loses 1.375 kg in the water ; 5 kg of silver loses 0.375 kg . what is the ratio of tin to silver in the bar ?\\n\"\n",
    "\"Options: a ) 1 / 4 , b ) 2 / 5 , c ) 1 / 2 , d ) 3 / 5 , e ) 2 / 3\\n\"\n",
    "\"Let's think step by step\\n\"\n",
    "\"you can simply use this formula to avoid confusion: w1 / w2 = ( a2 - aavg ) / ( avg - a1 )\\n\",\n",
    "\"here is how you will find the values of a1 an a2.\\n\", \n",
    "\"we have an overall loss ( average loss ).\\n\",\n",
    "\"the average loss is 6 kg when 60 kg alloy is immersed.\\n\", \n",
    "\"this is a loss of ( 6 / 60 ) * 100 = 10 %.\\n\",\n",
    "\"this is aavg the loss of tin is 1.375 kg for every 10 kg.\\n\", \n",
    "\"this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water.\\n\",\n",
    "\"this is a1.\\n\",\n",
    "\"the loss of silver is .375 kg for every 5 kg.\\n\", \n",
    "\"this means it loses ( .375 / 5 ) * 100 = 7.5 % of its weight in water.\\n\",\n",
    "\"this is a2.\\n\",\n",
    "\"weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss )\\n\",\n",
    "\"x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3\\n\",\n",
    "\"The answer is e, 2 / 3\\n\\n\",\n",
    "    \n",
    "\n",
    "\"Question: a man has $ 10,000 to invest . he invests $ 4000 at 5 % and $ 3500 at 4 % . in order to have a yearly income of $ 500 , he must invest the remainder at :\\n\",\n",
    "\"Options: a ) 4.6 % , b ) 3.6 % , c ) 6.4 % , d ) 7.8 % , e ) 8.9 %\\n\",\n",
    "\"Let's think step by step\\n\"\n",
    "\"income from $4000 at 5% in one year = $4000 of 5% = $4000 × 5 / 100 = $4000 × 0.05 = $200.\\n\",\n",
    "\"income from $ 3500 at 4 % in one year = $3500 of 4% = $3500 × 4 / 100 = $3500 × 0.04 = $140.\\n\",\n",
    "\"total income from 4000 at 5% and 3500 at 4% = $200 + $140 = $340.\\n\",\n",
    "\"remaining income amount in order to have a yearly income of $500 is $500 - $340 = $160.\\n\",\n",
    "\"total invested amount = $4000 + $3500 = $7500.\\n\",\n",
    "\"remaining invest amount = $10000 - $7500 = $2500.\\n\",\n",
    "\"we know that, interest = principal × rate × time interest = $160\\n\",\n",
    "\"principal = $2500 , rate = r\\n\",\n",
    "\"we need to find the value of r\\n\",\n",
    "\"time = 1 year . 160 = 2500 × r × 1\\n\", \n",
    "\"160 = 2500r\\n\", \n",
    "\"160 / 2500 = 2500 r / 2500 (divide both sides by 2500)\\n\",\n",
    "\"r = 0.064\\n\",\n",
    "\"change it to a percent by moving the decimal to the right two places r = 6.4%\\n\", \n",
    "\"therefore , he invested the remaining amount $2500 at 6.4% in order to get $500 income every year.\\n\", \n",
    "\"The answer is c, 6.4%\\n\\n\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "36b5c320-82a6-441a-ab06-5cb26da11986",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: the difference in compound interest earned on a deposit ( compounded annually ) in year 1 and year 2 is $ 40 . had the interest rate been 3 times its present value , the difference c would have been how much ?\n",
      "Options: a ) 40 / 3 , b ) 40 , c ) 120 , d ) 360 , e ) 420\n",
      "Let's think step by step\n",
      "case 1 : deposit = $x\n",
      "rate of increase = r\n",
      "interest yearned in 1 year = xr\n",
      "deposit in 1 year = x + xr\n",
      "interest yearned in 2 year = ( x + xr ) r\n",
      "the difference c = ( x + xr ) r - xr = xr ^ 2 = 40\n",
      "case 2 : deposit = $x\n",
      ",rate of increase = 3r\n",
      "interest yearned in 1 year = x ( 3 r )\n",
      "deposit in 1 year = x + 3 xr\n",
      "interest yearned in 2 year = ( x + 3 xr ) 3 r\n",
      "the difference = ( x + 3 xr ) 3 r - 3 xr = 9 xr ^ 2\n",
      "since from case 1 we know that xr ^ 2 = 40\n",
      "then 9 xr ^ 2 = 9 * 40 = 360.\n",
      "The answer is d, 360\n",
      "\n",
      "Question: a man invests rs . 8,000 at the rate of 5 % per annum . how much more should he invest at the rate of 8 % , so that he can earn a total of 6 % per annum ?\n",
      "Options: a ) rs . 1200 , b ) rs . 1300 , c ) rs . 1500 , d ) rs . 4000 , e ) none of these\n",
      "Let's think step by step\n",
      "interest on rs . 8000 at 5 % per annum = ( 8000 × 5 × 1 ) / 100 = rs . 400\n",
      "let his additional investment at 8 % = x\n",
      "interest on rs . x at 8 % per annum = ( x × 8 × 1 ) / 100 = 2 x / 25\n",
      "to earn 6 % per annum for the total, interest = ( 8000 + x ) × 6 × 1 / 100.\n",
      "400 + 2 x / 25 = ( 8000 + x ) × 6 × 1 / 100.\n",
      "40000 + 8 x = ( 8000 + x ) × 6\n",
      "40000 + 8 x = 48000 + 6 x\n",
      "2 x = 8000. x = 4000.\n",
      "The answer is d, rs . 4000\n",
      "\n",
      "Question: mr . john used to purchase certain number of mangoes for $ 360 since the price of mangoes is reduced by 10 % he got 12 more mangoes today . find the original price of 125 mangoes.\n",
      "Options: a ) 360 , b ) 380 , c ) 400 , d ) 416 , e ) 412\n",
      "Let's think step by step\n",
      "mr. john used to purchase certain number of mangoes for $ 360\n",
      "since the price of mangoes is reduced by 10 %\n",
      "he got 12 more mangoes today.\n",
      "find the original price of 120 mangoes.\n",
      "method 1 : let price per mango = x.\n",
      "let number of mangoes be n.\n",
      "then , nx = 360.\n",
      "now price = 0.9 x\n",
      "number of mangoes = n + 12\n",
      "total amount = 0.9 x * ( n + 12 ) = 360\n",
      "nx = 0.9 nx + 10.8 x => 0.1 nx = 10.8 x => n = 108\n",
      "x = 360 / 108 = 3.33\n",
      "original price of 125 mangoes = 125 * 3.33 = 416.\n",
      "The answer is d, 416\n",
      "\n",
      "Question: tim and é lan are 90 miles away from one another . they are starting to move towards each other simultaneously , tim at a speed of 10 mph and é lan at a speed of 5 mph . if every hour they double their speeds , what is the distance that tim will pass until he meets é lan ?\n",
      "Options: a ) 30 miles . , b ) 35 miles . , c ) 45 miles . , d ) 60 miles . , e ) 65 miles .\n",
      "Let's think step by step\n",
      "in 1 st hour: relative speed of the two will be 10 + 5 = 15 miles / hour\n",
      "as they move towards each other, the distance between them reduces by 15 miles.\n",
      "distance left between them: 90 - 15 = 75 miles.\n",
      "in 2 d hour: speed of tim: 20 mph, that of elan : 10 mph.\n",
      "relative speed of the two will be 20 + 10 = 30 miles / hour.\n",
      "as they move towards each other , the distance between them reduces by 30 miles.\n",
      "distance left between them : 75 - 30 = 45 miles.\n",
      "in 3 rd hour: speed of tim: 40 mph , that of elan: 20 mph.\n",
      "relative speed of the two will be 40 + 20 = 60 miles / hour.\n",
      "now in order to cover the remaining distance of 45 miles . they will take : ( 45 / 60 ) hrs . = ( 3 / 4 ) hour.\n",
      "distance travelled by tim in first hour : 1 hour * ( 10 miles per hour ) = 10 miles\n",
      "in second hour : 1 hour * ( 20 miles per hour ) = 20 miles\n",
      "in third hour : ( 3 / 4 ) hour * ( 40 miles per hour ) = 30 miles\n",
      "total distance travelled : 10 + 20 + 30 = 60 miles.\n",
      "The answer is d, 60 miles.\n",
      "\n",
      "Question: what is the sum of all 3 digit numbers that leave a remainder of ' 2 ' when divided by 3 ?\n",
      "Options: a ) 897 , b ) 164,850 , c ) 164,749 , d ) 149,700 , e ) 156,720\n",
      "Let's think step by step\n",
      "step 1 : identify the series the smallest 3 digit number that will leave a remainder of 2 when divided by 3 is 101.\n",
      "the next couple of numbers that will leave a remainder of 2 when divided by 3 are 104 and 107.\n",
      "the largest 3 digit number that will leave a remainder of 2 when divided by 3 is 998.\n",
      "it is evident that any number in the sequence will be a 3 digit positive integer of the form ( 3 n + 2 ).\n",
      "so , the given numbers are in an arithmetic sequence with the first term being 101 and the last term being 998 and the common difference being 3.\n",
      "step 2 : compute the sum sum of an arithmetic progression ( ap ) = [ first term + last term / 2 ] n \n",
      "we know the first term : 101\n",
      "we know the last term : 998\n",
      "the only unknown is the number of terms , n .\n",
      "in an a . p . , the nth term an = a 1 + ( n - 1 ) * d\n",
      "in this case , therefore , 998 = 101 + ( n - 1 ) * 3 or 897 = ( n - 1 ) * 3 ( n - 1 ) = 299 or n = 300.\n",
      "sum of the ap will therefore , be [ 101 + 998 / 2 ] ∗ 300 = 164,850\n",
      "The answer is b, 164,850\n",
      "\n",
      "Question: what is the area inscribed by the lines y = 10 , x = 1 , y = 3 x + 1 on an xy - coordinate plane ?\n",
      "Options: a ) 1 , b ) 2 , c ) 3 , d ) 6 , e ) 12\n",
      "Let's think step by step\n",
      "let' s graph the lines y = 10 and x = 1 at this point\n",
      "we need to find the points where the line y = 3 x + 1 intersects the other two lines.\n",
      "for the vertical line, we know that x = 1\n",
      "so we' ll plug x = 1 into the equation y = 3 x + 1 to get y = 3 ( 1 ) + 1 = 4\n",
      "when x = 1, y = 5, so one point of intersection is (1, 4)\n",
      "for the horizontal line , we know that y = 10\n",
      "so we' ll plug y = 10 into the equation y = 3x + 1 to get 10 = 3x + 1.\n",
      "solve to get: x = 3\n",
      "so, when y = 10 , x = 3, so one point of intersection is ( 3,10 ) now add these points to our graph\n",
      "and sketch the line y = 3 x + 1 at this point, we can see that we have the following triangle.\n",
      "the base has length 2 and the height is 6 area = (1 / 2) ( base ) ( height ) = ( 1 / 2 ) ( 2 ) ( 6 ) = 6\n",
      "The answer is d, 6\n",
      "\n",
      "Question: a 60 kg metal bar made of alloy of tin and silver lost 6 kg of its weight in the water . 10 kg of tin loses 1.375 kg in the water ; 5 kg of silver loses 0.375 kg . what is the ratio of tin to silver in the bar ?\n",
      "Options: a ) 1 / 4 , b ) 2 / 5 , c ) 1 / 2 , d ) 3 / 5 , e ) 2 / 3\n",
      "Let's think step by step\n",
      "you can simply use this formula to avoid confusion: w1 / w2 = ( a2 - aavg ) / ( avg - a1 )\n",
      "here is how you will find the values of a1 an a2.\n",
      "we have an overall loss ( average loss ).\n",
      "the average loss is 6 kg when 60 kg alloy is immersed.\n",
      "this is a loss of ( 6 / 60 ) * 100 = 10 %.\n",
      "this is aavg the loss of tin is 1.375 kg for every 10 kg.\n",
      "this means it loses ( 1.375 / 10 ) * 100 = 13.75 % of its weight in water.\n",
      "this is a1.\n",
      "the loss of silver is .375 kg for every 5 kg.\n",
      "this means it loses ( .375 / 5 ) * 100 = 7.5 % of its weight in water.\n",
      "this is a2.\n",
      "weight of tin / weight of silver = ( silver ' s loss - avg loss ) / ( avg loss - tin ' s loss )\n",
      "x / y = ( 7.5 - 10 ) / ( 10 - 13.75 ) = 2 / 3\n",
      "The answer is e, 2 / 3\n",
      "\n",
      "Question: a man has $ 10,000 to invest . he invests $ 4000 at 5 % and $ 3500 at 4 % . in order to have a yearly income of $ 500 , he must invest the remainder at :\n",
      "Options: a ) 4.6 % , b ) 3.6 % , c ) 6.4 % , d ) 7.8 % , e ) 8.9 %\n",
      "Let's think step by step\n",
      "income from $4000 at 5% in one year = $4000 of 5% = $4000 × 5 / 100 = $4000 × 0.05 = $200.\n",
      "income from $ 3500 at 4 % in one year = $3500 of 4% = $3500 × 4 / 100 = $3500 × 0.04 = $140.\n",
      "total income from 4000 at 5% and 3500 at 4% = $200 + $140 = $340.\n",
      "remaining income amount in order to have a yearly income of $500 is $500 - $340 = $160.\n",
      "total invested amount = $4000 + $3500 = $7500.\n",
      "remaining invest amount = $10000 - $7500 = $2500.\n",
      "we know that, interest = principal × rate × time interest = $160\n",
      "principal = $2500 , rate = r\n",
      "we need to find the value of r\n",
      "time = 1 year . 160 = 2500 × r × 1\n",
      "160 = 2500r\n",
      "160 / 2500 = 2500 r / 2500 (divide both sides by 2500)\n",
      "r = 0.064\n",
      "change it to a percent by moving the decimal to the right two places r = 6.4%\n",
      "therefore , he invested the remaining amount $2500 at 6.4% in order to get $500 income every year.\n",
      "The answer is c, 6.4%\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join(prompt_hard_2_lb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "7a33929d-76cd-47c9-98b1-8357e19ceb26",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [22:16<00:00,  6.68s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_coht_hard_2_lb.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = {}\n",
    "        for ac in c.split(', '):\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_hard_2_lb) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "10fe24c1-204d-419d-a7be-a56f35bf7a8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 62 ratio 0.3100\n"
     ]
    }
   ],
   "source": [
    "_, ans_pred_hard_1, ans_gold_hard_1 =parse_pred_ans('outputs/dev_outputs_coht_hard_2_lb.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d336395-21ca-48f4-90df-c3e310196448",
   "metadata": {},
   "source": [
    "## Random CoT, Acc 33.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "id": "4d762fe9-a4af-4b3d-a4b6-383b02648811",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_random = open('MathQA/prompt_random.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "id": "ceab0dae-bab1-42fb-9aee-f0be84005a73",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [16:09<00:00,  4.85s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_random.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_random) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "id": "b4a19eba-993f-4e6c-9c0a-d821413ce4be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 66 ratio 0.3300\n"
     ]
    }
   ],
   "source": [
    "_, _, _ =parse_pred_ans('outputs/dev_outputs_random.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "067a1f4e-a1df-4c17-a456-81b8398a67a7",
   "metadata": {},
   "source": [
    "## Retrieval, Acc 69.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "id": "9fbe39e1-11f9-407d-8bdf-13170c79bf6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Problem': \"the banker ' s gain of a certain sum due 3 years hence at 10 % per annum is rs . 36 . what is the present worth ?\",\n",
       " 'Rationale': '\"explanation : t = 3 years r = 10 % td = ( bg × 100 ) / tr = ( 36 × 100 ) / ( 3 × 10 ) = 12 × 10 = rs . 120 td = ( pw × tr ) / 100 ⇒ 120 = ( pw × 3 × 10 ) / 100 ⇒ 1200 = pw × 3 pw = 1200 / 3 = rs . 400 answer : option a\"',\n",
       " 'options': 'a ) rs . 400 , b ) rs . 300 , c ) rs . 500 , d ) rs . 350 , e ) none of these',\n",
       " 'correct': 'a',\n",
       " 'annotated_formula': 'divide(multiply(const_100, divide(multiply(36, const_100), multiply(3, 10))), multiply(3, 10))',\n",
       " 'linear_formula': 'multiply(n2,const_100)|multiply(n0,n1)|divide(#0,#1)|multiply(#2,const_100)|divide(#3,#1)|',\n",
       " 'category': 'gain'}"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "id": "63bdd90d-765f-4ad5-bf65-1ff321cf8005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Problem': 'a man goes from a to b at a speed of 20 kmph and comes back to a at a speed of 30 kmph . find his average speed for the entire journey ?',\n",
       " 'Rationale': '\"distance from a and b be \\' d \\' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 20 ) + ( d / 30 ) ] = ( 2 d ) / [ 5 d / 60 ) = > 24 kmph . answer : b\"',\n",
       " 'options': 'a ) 11 , b ) 24 , c ) 55 , d ) 66 , e ) 71',\n",
       " 'correct': 'b',\n",
       " 'annotated_formula': 'divide(add(20, 30), const_2)',\n",
       " 'linear_formula': 'add(n0,n1)|divide(#0,const_2)|',\n",
       " 'category': 'physics'}"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_data_200[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "id": "57a3ac6d-b75a-432d-a054-79bf41d25009",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('all-MiniLM-L6-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "id": "f5b8d1b9-c04b-4e30-9a78-9462c105d4ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_questions = [d['Problem'] for d in dev_data_200]\n",
    "train_questions = [d['Problem'] for d in train_data]\n",
    "\n",
    "embeddings_dev = model.encode(dev_questions, convert_to_tensor=True)\n",
    "embeddings_train = model.encode(train_questions, convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "id": "393a6f78-8768-4092-be3c-d1641f2c0339",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([200, 384])"
      ]
     },
     "execution_count": 398,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_dev.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "id": "bf994092-2235-4b7f-a58b-1610980ca63d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([29837, 384])"
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_train.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "id": "cdbe995b-7fcd-44b4-8706-6ca1fa2e799f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cosine_scores = util.cos_sim(embeddings_dev, embeddings_train)\n",
    "\n",
    "neighbor_index = []\n",
    "for i in range(len(dev_questions)):\n",
    "    ind = cosine_scores[i].topk(k=8).indices\n",
    "    neighbor_index.append(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "ffc048f4-e3ca-415a-bf9b-708b0d2d7c0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a man goes from a to b at a speed of 20 kmph and comes back to a at a speed of 30 kmph . find his average speed for the entire journey ?'"
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_questions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "f9de95df-6e2c-43d3-827a-16fec98977f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a man goes from a to b at a speed of 60 kmph and comes back to a at a speed of 40 kmph . find his average speed for the entire journey ?\n",
      "\n",
      "a man goes from a to b at a speed of 21 kmph and comes back to a at a speed of 21 kmph . find his average speed for the entire journey ?\n",
      "\n",
      "a man goes from a to b at a speed of 35 kmph and comes back to a at a speed of 28 kmph . find his average speed for the entire journey ?\n",
      "\n",
      "a man goes from a to b at a speed of 14 kmph and comes back to a at a speed of 10 kmph . find his average speed for the entire journey ?\n",
      "\n",
      "a man goes from a to b at a speed of 15 kmph and comes back to a at a speed of 12 kmph . find his average speed for the entire journey ?\n",
      "\n",
      "mike took a bus from home to market , that travels at 30 kmph . while walking back at 6 kmph , halfway through , he suddenly realized he was getting late and he cycled back the remaining distance in 20 kmph . find the average speed .\n",
      "\n",
      "the speed of a car is 20 km in the first hour and 30 km in the second hour . what is the average speed of the car ?\n",
      "\n",
      "a person travels from p to q a speed of 30 km / hr and returns by increasing his speed by 30 % . what is his average speed for both the trips ?\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in neighbor_index[0]: print(train_questions[i] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "id": "58bb70b3-796b-440f-9b83-627e39d0fe7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prompt(neighbor_index, max_len=10000):\n",
    "    prompt = ''\n",
    "    for i in neighbor_index:\n",
    "        \n",
    "        case = train_data[i]\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        s = '' \n",
    "        s += 'Question: ' + q + '\\n'\n",
    "        s += 'Options: ' + c + '\\n'\n",
    "        s += \"Let's think step by step\\n\"\n",
    "        s += a + '\\n'\n",
    "        s += 'The answer is ' + cr + ', ' + ans_dict[cr] + '\\n\\n'\n",
    "        \n",
    "        if(len(prompt) + len(s) > max_len): continue\n",
    "        \n",
    "        prompt += s\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "f6bb9502-a2db-4748-b5d7-3782af3b2a57",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = make_prompt(neighbor_index[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "id": "c3d9064e-874d-4fe2-b760-5b768b016fec",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a man goes from a to b at a speed of 60 kmph and comes back to a at a speed of 40 kmph . find his average speed for the entire journey ?\n",
      "Options: a ) 49 kmph , b ) 58 kmph , c ) 44 kmph , d ) 47 kmph , e ) 48 kmph\n",
      "Let's think step by step\n",
      "\"distance from a and b be ' d ' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 60 ) + ( d / 40 ] = ( 2 d ) / [ 5 d / 120 ) = > 48 kmph . answer : e\" 48 kmph\n",
      "The answer is e, 48 kmph\n",
      "\n",
      "Question: a man goes from a to b at a speed of 21 kmph and comes back to a at a speed of 21 kmph . find his average speed for the entire journey ?\n",
      "Options: a ) 23.3 kmph , b ) 25.3 kmph , c ) 22.5 kmph , d ) 22.3 kmph , e ) 22.9 kmph\n",
      "Let's think step by step\n",
      "\"distance from a and b be ' d ' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 21 ) + ( d / 24 ] = ( 2 d ) / [ 15 d / 168 ) = > 22.3 kmph . answer : d\" 22.3 kmph\n",
      "The answer is d, 22.3 kmph\n",
      "\n",
      "Question: a man goes from a to b at a speed of 35 kmph and comes back to a at a speed of 28 kmph . find his average speed for the entire journey ?\n",
      "Options: a ) 38.9 kmph , b ) 38.8 kmph , c ) 38.1 kmph , d ) 37.1 kmph , e ) 37.8 kmph\n",
      "Let's think step by step\n",
      "\"distance from a and b be ' d ' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 35 ) + ( d / 28 ] = ( 2 d ) / [ 11 d / 140 ) = > 38.1 kmph . answer : c\" 38.1 kmph\n",
      "The answer is c, 38.1 kmph\n",
      "\n",
      "Question: a man goes from a to b at a speed of 14 kmph and comes back to a at a speed of 10 kmph . find his average speed for the entire journey ?\n",
      "Options: a ) 11.6 kmph , b ) 12.9 kmph , c ) 12.8 kmph , d ) 11.9 kmph , e ) 12.6 kmph\n",
      "Let's think step by step\n",
      "\"distance from a and b be ' d ' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 14 ) + ( d / 10 ] = ( 2 d ) / [ 12 d / 70 ) = > 11.6 kmph . answer : a\" 11.6 kmph\n",
      "The answer is a, 11.6 kmph\n",
      "\n",
      "Question: a man goes from a to b at a speed of 15 kmph and comes back to a at a speed of 12 kmph . find his average speed for the entire journey ?\n",
      "Options: a ) 13.7 kmph , b ) 13.3 kmph , c ) 13.8 kmph , d ) 13.9 kmph , e ) 12.3 kmph\n",
      "Let's think step by step\n",
      "distance from a and b be ' d ' average speed = total distance / total time average speed = ( 2 d ) / [ ( d / 15 ) + ( d / 12 ] = ( 2 d ) / [ 9 d / 60 ) = > 13.3 kmph . answer : b 13.3 kmph\n",
      "The answer is b, 13.3 kmph\n",
      "\n",
      "Question: mike took a bus from home to market , that travels at 30 kmph . while walking back at 6 kmph , halfway through , he suddenly realized he was getting late and he cycled back the remaining distance in 20 kmph . find the average speed .\n",
      "Options: a ) 21.2 kmph , b ) 12.1 kmph , c ) 23.4 kmph , d ) 20.4 kmph , e ) none of these\n",
      "Let's think step by step\n",
      "let the distance be 2 x ( one way ) time taken by bus = 2 x / 30 , by walking = x / 6 , by cycling = x / 20 hours : . average speed = total distance / total time = 6 x / x / 15 + x / 6 + x / 20 = 6 * 60 / 4 + 10 + 3 = 21.2 answer : a 21.2 kmph\n",
      "The answer is a, 21.2 kmph\n",
      "\n",
      "Question: the speed of a car is 20 km in the first hour and 30 km in the second hour . what is the average speed of the car ?\n",
      "Options: a ) 25 kmph , b ) 85 kmph , c ) 34 kmph , d ) 23 kmph , e ) 45 kmph\n",
      "Let's think step by step\n",
      "\"s = ( 20 + 30 ) / 2 = 25 kmph answer : a\" 25 kmph\n",
      "The answer is a, 25 kmph\n",
      "\n",
      "Question: a person travels from p to q a speed of 30 km / hr and returns by increasing his speed by 30 % . what is his average speed for both the trips ?\n",
      "Options: a ) 33 , b ) 77 , c ) 48 , d ) 99 , e ) 34.5\n",
      "Let's think step by step\n",
      "\"speed on return trip = 130 % of 30 = 39 km / hr . average speed of trip = 30 + 39 / 2 = 69 / 2 = 34.5 km / hr answer : e\" 34.5\n",
      "The answer is e, 34.5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "id": "a149a02d-4b30-45b4-8116-4313cad924b1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [11:27<00:00,  3.44s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "\n",
    "with open('outputs/dev_outputs_retrieval.txt', 'w') as fd, open('outputs/dev_outputs_retrieval_neighbor.txt', 'w') as fd_neighbor:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        neighbor = neighbor_index[i]\n",
    "        prompt_retrieval = make_prompt(neighbor)\n",
    "        fd_neighbor.write(prompt_retrieval + '--------\\n\\n')\n",
    "        \n",
    "        prompt_full = prompt_retrieval + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "id": "d075b808-d857-46ce-80a4-aeb88a43e249",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 139 ratio 0.6950\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_retrieval.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a12c5ecb-1676-4267-be96-52ea2af07d14",
   "metadata": {},
   "source": [
    "## Centroid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "id": "80d6356d-156f-45f8-9807-1e6300975aea",
   "metadata": {},
   "outputs": [],
   "source": [
    "centroid_scores = util.cos_sim(embeddings_train, embeddings_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "id": "d5ee02ca-46ab-4a84-a16f-587b1f52844b",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_idx = centroid_scores.mean(dim=-1).topk(k=8).indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "id": "d96f690d-6dde-4a25-ab3b-3a1775753008",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_centroid = make_prompt(top_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "id": "3255c397-87fd-40b1-9628-a1331f037728",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: the amount of time that three people worked on a special project was in the ratio of 3 to 5 to 6 . if the project took 140 hours , how many more hours did the hardest working person work than the person who worked the least ?\n",
      "Options: a ) 32 hours , b ) 40 hours , c ) 35 hours , d ) 30 hours , e ) 36 hours\n",
      "Let's think step by step\n",
      "\"let the persons be a , b , c . hours worked : a = 3 * 140 / 14 = 30 hours b = 5 * 140 / 14 = 50 hours c = 6 * 140 / 14 = 60 hours c is the hardest worker and a worked for the least number of hours . so the difference is 60 - 30 = 30 hours . answer : d\" 30 hours\n",
      "The answer is d, 30 hours\n",
      "\n",
      "Question: the amount of time that three people worked on a special project was in the ratio of 4 to 3 to 5 . if the project took 120 hours , how many more hours did the hardest working person work than the person who worked the least ?\n",
      "Options: a ) 25 hours , b ) 30 hours , c ) 20 hours , d ) 26 hours , e ) 32 hours\n",
      "Let's think step by step\n",
      "\"let the persons be a , b , c . hours worked : a = 4 * 120 / 12 = 40 hours b = 3 * 120 / 12 = 30 hours c = 5 * 120 / 12 = 50 hours c is the hardest worker and b worked for the least number of hours . so the difference is 50 - 30 = 20 hours . answer : c\" 20 hours\n",
      "The answer is c, 20 hours\n",
      "\n",
      "Question: the amount of time that three people worked on a special project was in the ratio of 2 to 3 to 4 . if the project took 90 hours , how many more hours did the hardest working person work than the person who worked the least ?\n",
      "Options: a ) 18 hours , b ) 22 hours , c ) 25 hours , d ) 26 hours , e ) 20 hours\n",
      "Let's think step by step\n",
      "\"let the persons be a , b , c . hours worked : a = 2 * 90 / 9 = 20 hours b = 3 * 90 / 9 = 30 hours c = 4 * 90 / 9 = 40 hours c is the hardest worker and a worked for the least number of hours . so the difference is 40 - 20 = 20 hours . answer : e\" 20 hours\n",
      "The answer is e, 20 hours\n",
      "\n",
      "Question: a , b and c completed a piece of work , a worked for 6 days , b for 9 days and c for 4 days . their daily wages were in the ratio of 3 : 4 : 5 . find the daily wages of c , if their total earning was rs . 1406 ?\n",
      "Options: a ) s . 109 , b ) s . 108 , c ) s . 100 , d ) s . 95 , e ) s . 102\n",
      "Let's think step by step\n",
      "\"3 x 4 x 5 x 6 9 4 18 x + 36 x + 20 x = 1406 74 x = 1406 = > x = 19 5 x = 95 rs . answer : d\" s . 95\n",
      "The answer is d, s . 95\n",
      "\n",
      "Question: there is a train and car . the ratio between the speed of a train & a car is 16 : 15 respectively . also , a bus covered a distance of 480 km in 8 hours . the speed of the bus is 3 / 4 th of the speed of the train . how much distance will the car cover in 5 hours ?\n",
      "Options: a ) 300 km , b ) 325 km , c ) 375 km , d ) 425 km , e ) 450 km\n",
      "Let's think step by step\n",
      "\"the speed of the bus is 480 / 8 = 60 km / hr the speed of the train is ( 60 * 4 ) / 3 = 80 km / hr the speed of the car is 80 / 16 * 15 = 75 km / hr the distance covered by the car in 5 hours is 75 × 5 = 375 km the answer is c .\" 375 km\n",
      "The answer is c, 375 km\n",
      "\n",
      "Question: a , b and c completed a piece of work , a worked for 16 days , b for 9 days and c for 4 days . their daily wages were in the ratio of 3 : 4 : 5 . find the daily wages of c , if their total earning was rs . 1480 ?\n",
      "Options: a ) rs . 80 , b ) rs . 120 , c ) rs . 71.15 , d ) rs . 51.15 , e ) rs . 180\n",
      "Let's think step by step\n",
      "\"3 x 4 x 5 x 16 9 4 48 x + 36 x + 20 x = 1480 104 x = 1480 = > x = 14.23 5 x = 71.15 rs . answer : c\" rs . 71.15\n",
      "The answer is c, rs . 71.15\n",
      "\n",
      "Question: there is a train and car . the ratio between the speed of a train & a car is 16 : 15 respectively . also , a bus covered a distance of 480 km in 8 hours . the speed of the bus is 3 / 4 th of the speed of the train . how many kilometers will the car cover in 4 hours ?\n",
      "Options: a ) 200 , b ) 250 , c ) 300 , d ) 350 , e ) 400\n",
      "Let's think step by step\n",
      "\"the speed of the bus is 480 / 8 = 60 km / hr the speed of the train is ( 60 * 4 ) / 3 = 80 km / hr the speed of the car is 80 / 16 * 15 = 75 km / hr the distance covered by the car in 4 hours is 75 × 4 = 300 km the answer is c .\" 300\n",
      "The answer is c, 300\n",
      "\n",
      "Question: 60 % of a number is added to 160 , the result is the same number . find the number ?\n",
      "Options: a ) 300 , b ) 400 , c ) 266 , d ) 99 , e ) 121\n",
      "Let's think step by step\n",
      "\": ( 60 / 100 ) * x + 160 = x 2 x = 800 x = 400 answer : b\" 400\n",
      "The answer is b, 400\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_centroid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "id": "01561a03-7bec-452b-82f7-931c562e277d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [14:16<00:00,  4.28s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_centroid.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_centroid) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "id": "41059fc8-b4bb-4093-a97a-f66809a61f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 64 ratio 0.3200\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_centroid.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
